view fetchnews.php @ 2:40e545510a57

Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
author Ivo Smits <Ivo@UCIS.nl>
date Tue, 12 Apr 2011 00:29:41 +0200
parents 61fac319ca3e
children 0dcdb73cbcbf
line wrap: on
line source

<?php
chdir(__DIR__);
require_once './pdo.php';
require_once './config.php';

foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) {
	$socket = stream_socket_client($peer['address']);
	if ($socket === FALSE) die("Could not connect to peer $peer[address]\n");
	$line = nntp_readline($socket);
	$code = strtok($line, " \t");
	if ($code == 200) {
	} else if ($code == 201) {
		$peer['post'] = 0;
	} else die("Error code $code from $peer[address]\n");
	foreach ($db->evalAllAssoc('SELECT * FROM `peergroups` WHERE `peer` = ?', $peer['id']) as $peergroup) {
		$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `id` = ?', $peergroup['group']);
		nntp_writeline($socket, 'GROUP '.$group['name']);
		$line = nntp_readline($socket);
		$code = strtok($line, " \t");
		if ($code != 211) die("Error code $code from $peer[address]\n");
		strtok(" \t"); //number of articles
		$low = strtok(" \t");
		$high = strtok(" \t");
		strtok(" \t"); //group name
		if ($low != $peergroup['low'] || $high != $peergroup['high'] || $peergroup['low'] === NULL || $peergroup['high'] === NULL) {
			for ($i = $low; $i <= $high; $i++) {
				if ($i >= $peergroup['low'] && $i <= $peergroup['high'] && $peergroup['low'] !== NULL && $peergroup['high'] !== NULL) continue;
				nntp_writeline($socket, 'STAT '.$i);
				$line = nntp_readline($socket);
				$code = strtok($line, " \t");
				if ($code == 423) {
					print("Gap in article numbering at $i\n");
					continue;
				}
				if ($code != 223) die("Error code $code from $peer[address]\n");
				strtok(" \t"); //article number
				$messageid = strtok(" \t");
				if ($messageid[0] != '<' || $messageid[strlen($messageid)-1] != '>') die("Malformed message ID $messageid\n");
				$messageid = substr($messageid, 1, -1);
				$message = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid);
				if ($message) {
					$groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id']));
					if (!$groupmessage) {
						$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id']));
					}
				} else {
					nntp_writeline($socket, 'HEAD '.$i);
					$line = nntp_readline($socket);
					$code = strtok($line, " \t");
					if ($code != 221) die("Error code $code from $peer[address]\n");
					strtok(" \t"); //article number
					$lines = nntp_readlines($socket);
					nntp_removeheader(&$lines, 'Xref');
					nntp_updatepath(&$lines, 'pNewss.Core.UCIS.nl');
					$header = implode("\r\n", $lines);
					
					nntp_writeline($socket, 'BODY '.$i);
					$line = nntp_readline($socket);
					$code = strtok($line, " \t");
					if ($code != 222) die("Error code $code from $peer[address]\n");
					strtok(" \t"); //article number
					$lines = nntp_readlines($socket);
					$body = implode("\r\n", $lines);
					
					$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, $header, $body));
					$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
				}
			}
			$db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group']));
		}
	}
	while ($peer['post']) {
		if ($peer['lastposted'] === NULL) {
			$articles = $db->evalAllAssoc('SELECT * FROM `messages` LIMIT 10');
		} else {
			$articles = $db->evalAllAssoc('SELECT * FROM `messages` WHERE `id` > ? LIMIT 10', $peer['lastposted']);
		}
		if (!count($articles)) break;
		foreach ($articles as $article) {
			nntp_writeline($socket, 'POST');
			$line = nntp_readline($socket);
			$code = strtok($line, " \t");
			if ($code != 340) die("Error code $code from $peer[address]\n");
			foreach (explode("\r\n", $article['header']) as $line) {
				$parts = explode(': ', $line, 2);
				switch (strtoupper($parts[0])) {
					case 'PATH': case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION':
					case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING':
					case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO':
					case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
					case 'IN-REPLY-TO':
						break;
					case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
					case 'NNTP-POSTING-DATE':
						$line = NULL;
						break;
					default:
						print("Sending unknown header $parts[0]\n");
				}
				if ($line !== NULL) nntp_writeline($socket, $line);
			}
			nntp_writeline($socket, '');
			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline($socket, $line);
			nntp_writeline($socket, '.');
			$line = nntp_readline($socket);
			$code = strtok($line, " \t");
			if ($code != 240) print("Article $article[messageid] was not accepted ($code)\n");
			if ($article['id'] > $peer['lastposted']) $peer['lastposted'] = $article['id'];
		}
		$db->update('UPDATE `peers` SET `lastposted` = ? WHERE `id` = ?', array($peer['lastposted'], $peer['id']));
	}
	nntp_writeline($socket, 'QUIT');
	fclose($socket);
}

function nntp_removeheader(&$lines, $header) {
	$header = strtoupper($header).':';
	$hlen = strlen($header);
	foreach ($lines as $key => $line) if (strtoupper(substr($line, 0, $hlen)) == $header) unset($lines[$key]);
}
function nntp_updatepath(&$lines, $value) {
	$header = strtoupper('Path').':';
	$hlen = strlen($header);
	$found = FALSE;
	foreach ($lines as &$line) if (strtoupper(substr($line, 0, $hlen)) == $header) {
		$parts = explode(': ', $line, 2);
		$parts[1] = $value.'!'.$parts[1];
		$line = implode(': ', $parts);
		$found = TRUE;
		break;
	}
	if (!$found) $lines[] = 'Path: '.$value;
}

function nntp_readline($socket) {
	$line = rtrim(fgets($socket, 512), "\r\n");
	print('R: '.$line."\n");
	return $line;
}
function nntp_writeline($socket, $line) {
	print('W: '.$line."\n");
	fwrite($socket, $line."\r\n");
}
function nntp_readlines($socket) {
	$line = nntp_readline($socket);
	$lines = array();
	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
		$lines[] = $line;
		$line = nntp_readline($socket);
	}
	if ($line != '.') die("Unexpected end of message header\n");
	return $lines;
}