Mercurial > hg > pnewss

changeset 2:40e545510a57

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
author Ivo Smits <Ivo@UCIS.nl>
date Tue, 12 Apr 2011 00:29:41 +0200
parents 61fac319ca3e
children 0dcdb73cbcbf
files config.php.example database.mysql fetchnews.php readme.txt server.php todo.txt
diffstat 6 files changed, 126 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/config.php.example	Mon Apr 11 23:17:27 2011 +0200
+++ b/config.php.example	Tue Apr 12 00:29:41 2011 +0200
@@ -1,2 +1,5 @@
 <?php
 $db = PDOWrapper::connectMysql('localhost', 'dbname', 'username', 'password');
+//For other databases (see also http://nl2.php.net/manual/en/pdo.construct.php):
+//$db = new PDOWrapper($dsn, $user = NULL, $pass = NULL, $options = NULL);
+
--- a/database.mysql	Mon Apr 11 23:17:27 2011 +0200
+++ b/database.mysql	Tue Apr 12 00:29:41 2011 +0200
@@ -3,7 +3,7 @@
 -- http://www.phpmyadmin.net
 --
 -- Machine: localhost
--- Genereertijd: 11 Apr 2011 om 23:15
+-- Genereertijd: 12 Apr 2011 om 00:18
 -- Serverversie: 5.0.51
 -- PHP-Versie: 5.3.3-7
 
@@ -78,6 +78,8 @@
 CREATE TABLE IF NOT EXISTS `peers` (
   `id` int(10) unsigned NOT NULL auto_increment,
   `address` varchar(255) collate utf8_unicode_ci NOT NULL,
+  `post` tinyint(1) unsigned NOT NULL,
+  `lastposted` int(10) unsigned default NULL,
   PRIMARY KEY  (`id`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 
--- a/fetchnews.php	Mon Apr 11 23:17:27 2011 +0200
+++ b/fetchnews.php	Tue Apr 12 00:29:41 2011 +0200
@@ -1,4 +1,5 @@
 <?php
+chdir(__DIR__);
 require_once './pdo.php';
 require_once './config.php';
 
@@ -7,7 +8,10 @@
 	if ($socket === FALSE) die("Could not connect to peer $peer[address]\n");
 	$line = nntp_readline($socket);
 	$code = strtok($line, " \t");
-	if ($code != 200) die("Error code $code from $peer[address]\n");
+	if ($code == 200) {
+	} else if ($code == 201) {
+		$peer['post'] = 0;
+	} else die("Error code $code from $peer[address]\n");
 	foreach ($db->evalAllAssoc('SELECT * FROM `peergroups` WHERE `peer` = ?', $peer['id']) as $peergroup) {
 		$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `id` = ?', $peergroup['group']);
 		nntp_writeline($socket, 'GROUP '.$group['name']);
@@ -65,6 +69,46 @@
 			$db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group']));
 		}
 	}
+	while ($peer['post']) {
+		if ($peer['lastposted'] === NULL) {
+			$articles = $db->evalAllAssoc('SELECT * FROM `messages` LIMIT 10');
+		} else {
+			$articles = $db->evalAllAssoc('SELECT * FROM `messages` WHERE `id` > ? LIMIT 10', $peer['lastposted']);
+		}
+		if (!count($articles)) break;
+		foreach ($articles as $article) {
+			nntp_writeline($socket, 'POST');
+			$line = nntp_readline($socket);
+			$code = strtok($line, " \t");
+			if ($code != 340) die("Error code $code from $peer[address]\n");
+			foreach (explode("\r\n", $article['header']) as $line) {
+				$parts = explode(': ', $line, 2);
+				switch (strtoupper($parts[0])) {
+					case 'PATH': case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION':
+					case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING':
+					case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO':
+					case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
+					case 'IN-REPLY-TO':
+						break;
+					case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
+					case 'NNTP-POSTING-DATE':
+						$line = NULL;
+						break;
+					default:
+						print("Sending unknown header $parts[0]\n");
+				}
+				if ($line !== NULL) nntp_writeline($socket, $line);
+			}
+			nntp_writeline($socket, '');
+			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline($socket, $line);
+			nntp_writeline($socket, '.');
+			$line = nntp_readline($socket);
+			$code = strtok($line, " \t");
+			if ($code != 240) print("Article $article[messageid] was not accepted ($code)\n");
+			if ($article['id'] > $peer['lastposted']) $peer['lastposted'] = $article['id'];
+		}
+		$db->update('UPDATE `peers` SET `lastposted` = ? WHERE `id` = ?', array($peer['lastposted'], $peer['id']));
+	}
 	nntp_writeline($socket, 'QUIT');
 	fclose($socket);
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/readme.txt	Tue Apr 12 00:29:41 2011 +0200
@@ -0,0 +1,9 @@
+To use:
+	Set up some database (MySQL is recommended, but others should work too), with a schema as described in database.mysql
+	Rename/copy config.php.example to config.php and edit it according to your database settings
+	Add other news servers to the peers table in the database (address is in the format tcp://hostname:119, post is either 0 or 1 to try to post to this server)
+	Add newsgroups to the newsgroups table in the database
+	Add peer-group relations to the peergroups table in the database (peer is the id number from the peers table, group is the id number from the groups table)
+	Setup (x)inetd (or socat or netcat) to serve the server.php script on TCP port 119
+	Run the fetchnews.php script to synchronize with your peers
+	Install the fetchnews.php script in crontab to automatically synchronize with your peers
--- a/server.php	Mon Apr 11 23:17:27 2011 +0200
+++ b/server.php	Tue Apr 12 00:29:41 2011 +0200
@@ -1,6 +1,6 @@
 #!/usr/bin/php
 <?php
-chdir('/home/ivo/projects/pnewss');
+chdir(__DIR__);
 require_once './pdo.php';
 require_once './config.php';
 
@@ -24,7 +24,7 @@
 nntp_writeline(STDOUT, '200 pNewss ready');
 while (TRUE) {
 	$line = nntp_readline(STDIN);
-	if ($line === FALSE || $line === NULL) break;	
+	if ($line === FALSE || $line === NULL) break;
 	$cmd = strtok($line, " \t");
 	switch (strtoupper($cmd)) {
 		case 'LIST':
@@ -116,6 +116,66 @@
 				nntp_writeline(STDOUT, '223 '.$article['number'].' <'.$articlea['messageid'].'> ok');
 			}
 			break;
+		case 'POST':
+			nntp_writeline(STDOUT, '340 Ok, recommended message-ID <'.md5(time().rand()).'@pNews.Core.UCIS.nl>');
+			$lines = nntp_readlines(STDIN);
+			$header = array();
+			$headers = array();
+			while (count($lines)) {
+				$line = array_shift($lines);
+				if (!strlen($line)) break;
+				$parts = explode(': ', $line, 2);
+				$headers[strtoupper($parts[0])] = $parts[1];
+				switch (strtoupper($parts[0])) {
+					case 'PATH':
+						$line = $parts[0].': pNews.Core.UCIS.nl!'.$parts[1];
+						break;
+					case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION':
+					case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING':
+					case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO':
+					case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
+					case 'IN-REPLY-TO':
+						break;
+					case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
+					case 'NNTP-POSTING-DATE':
+						$line = NULL;
+						break;
+					default:
+						writelog("Received unknown header $parts[0]");
+				}
+				if ($line !== NULL) $header[] = $line;
+			}
+			if (!isset($headers['NEWSGROUPS'])) {
+				nntp_writeline(STDOUT, '441 Missing required Newsgroups: header');
+				break;
+			}
+			if (!isset($headers['MESSAGE-ID'])) {
+				$headers['MESSAGE-ID'] = '<'.md5(time().rand()).'@pNews.Core.UCIS.nl>';
+				$header[] = 'Message-ID: '.$headers['MESSAGE-ID'];
+			}
+			if (!isset($headers['PATH'])) {
+				$headers['PATH'] = 'pNews.Core.UCIS.nl';
+				$header[] = 'Path: '.$headers['PATH'];
+			}
+			$msgid = $headers['MESSAGE-ID'];
+			if (strlen($msgid) <= 2 || $msgid[0] != '<' || $msgid[strlen($msgid)-1] != '>') {
+				nntp_writeline(STDOUT, '441 435 Bad Message-ID');
+				break;
+			}
+			$msgid = substr($msgid, 1, -1);
+			$article = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $msgid);
+			if ($article !== FALSE) {
+				nntp_writeline(STDOUT, '441 435 Duplicate');
+				return NULL;
+			}
+			$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($msgid, implode("\r\n", $header), implode("\r\n", $lines)));
+			foreach (explode(',', $headers['NEWSGROUPS']) as $groupname) {
+				$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `name` = ?', $groupname);
+				if ($group === FALSE) continue;
+				$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
+			}
+			nntp_writeline(STDOUT, '240 Article received <'.$msgid.'>');
+			break;
 		case 'QUIT':
 			nntp_writeline(STDOUT, '205 .');
 			return;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/todo.txt	Tue Apr 12 00:29:41 2011 +0200
@@ -0,0 +1,4 @@
+- Make sure that group article numbers are never reused, not even if the last one is deleted (groupmessages table)
+- Support IHAVE command to speed up synchronization
+- Check Message-Id header in articles received from peers
+- Handle received cross-posted messages (according to Newsgroups header)