Mercurial > hg > pnewss

changeset 3:0dcdb73cbcbf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
author Ivo Smits <Ivo@UCIS.nl>
date Tue, 12 Apr 2011 01:47:09 +0200
parents 40e545510a57
children dc5a8e316e21
files database.mysql fetchnews.php reloadarticle.php server.php
diffstat 4 files changed, 64 insertions(+), 183 deletions(-) [+]
line wrap: on
line diff
--- a/database.mysql	Tue Apr 12 00:29:41 2011 +0200
+++ b/database.mysql	Tue Apr 12 01:47:09 2011 +0200
@@ -3,7 +3,7 @@
 -- http://www.phpmyadmin.net
 --
 -- Machine: localhost
--- Genereertijd: 12 Apr 2011 om 00:18
+-- Genereertijd: 12 Apr 2011 om 01:45
 -- Serverversie: 5.0.51
 -- PHP-Versie: 5.3.3-7
 
@@ -50,7 +50,7 @@
   `id` int(10) unsigned NOT NULL auto_increment,
   `messageid` varchar(255) character set ascii NOT NULL,
   `header` text character set ascii NOT NULL,
-  `body` text character set ascii NOT NULL,
+  `body` mediumtext character set ascii NOT NULL,
   PRIMARY KEY  (`id`),
   UNIQUE KEY `messageid` (`messageid`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
--- a/fetchnews.php	Tue Apr 12 00:29:41 2011 +0200
+++ b/fetchnews.php	Tue Apr 12 01:47:09 2011 +0200
@@ -1,7 +1,7 @@
+#!/usr/bin/php
 <?php
 chdir(__DIR__);
-require_once './pdo.php';
-require_once './config.php';
+require_once './common.php';
 
 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) {
 	$socket = stream_socket_client($peer['address']);
@@ -44,26 +44,17 @@
 						$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id']));
 					}
 				} else {
-					nntp_writeline($socket, 'HEAD '.$i);
+					nntp_writeline($socket, 'ARTICLE '.$i);
 					$line = nntp_readline($socket);
 					$code = strtok($line, " \t");
-					if ($code != 221) die("Error code $code from $peer[address]\n");
+					if ($code != 220) die("Error code $code from $peer[address]\n");
 					strtok(" \t"); //article number
 					$lines = nntp_readlines($socket);
-					nntp_removeheader(&$lines, 'Xref');
-					nntp_updatepath(&$lines, 'pNewss.Core.UCIS.nl');
-					$header = implode("\r\n", $lines);
-					
-					nntp_writeline($socket, 'BODY '.$i);
-					$line = nntp_readline($socket);
-					$code = strtok($line, " \t");
-					if ($code != 222) die("Error code $code from $peer[address]\n");
-					strtok(" \t"); //article number
-					$lines = nntp_readlines($socket);
-					$body = implode("\r\n", $lines);
-					
-					$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, $header, $body));
-					$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
+					try {
+						nntp_article_store($lines);
+					} catch (Exception $ex) {
+						writelog($ex->getMessage());
+					}
 				}
 			}
 			$db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group']));
@@ -113,41 +104,6 @@
 	fclose($socket);
 }
 
-function nntp_removeheader(&$lines, $header) {
-	$header = strtoupper($header).':';
-	$hlen = strlen($header);
-	foreach ($lines as $key => $line) if (strtoupper(substr($line, 0, $hlen)) == $header) unset($lines[$key]);
-}
-function nntp_updatepath(&$lines, $value) {
-	$header = strtoupper('Path').':';
-	$hlen = strlen($header);
-	$found = FALSE;
-	foreach ($lines as &$line) if (strtoupper(substr($line, 0, $hlen)) == $header) {
-		$parts = explode(': ', $line, 2);
-		$parts[1] = $value.'!'.$parts[1];
-		$line = implode(': ', $parts);
-		$found = TRUE;
-		break;
-	}
-	if (!$found) $lines[] = 'Path: '.$value;
+function writelog($line) {
+	print($line."\n");
 }
-
-function nntp_readline($socket) {
-	$line = rtrim(fgets($socket, 512), "\r\n");
-	print('R: '.$line."\n");
-	return $line;
-}
-function nntp_writeline($socket, $line) {
-	print('W: '.$line."\n");
-	fwrite($socket, $line."\r\n");
-}
-function nntp_readlines($socket) {
-	$line = nntp_readline($socket);
-	$lines = array();
-	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
-		$lines[] = $line;
-		$line = nntp_readline($socket);
-	}
-	if ($line != '.') die("Unexpected end of message header\n");
-	return $lines;
-}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/reloadarticle.php	Tue Apr 12 01:47:09 2011 +0200
@@ -0,0 +1,26 @@
+#!/usr/bin/php
+<?php
+chdir(__DIR__);
+require_once './common.php';
+
+if (count($_SERVER['argv']) < 2) die('Not enough arguments (usage: [server] [messageid])');
+$messageid = $_SERVER['argv'][2];
+$db->update('DELETE FROM `messages` WHERE `messageid` = ?', $messageid);
+$socket = stream_socket_client($_SERVER['argv'][1]);
+if ($socket === FALSE) die("Could not connect to peer\n");
+$line = nntp_readline($socket);
+$code = strtok($line, " \t");
+if ($code == 200) {
+} else if ($code == 201) {
+} else die("Error code $code\n");
+nntp_writeline($socket, 'ARTICLE <'.$messageid.'>');
+$line = nntp_readline($socket);
+$code = strtok($line, " \t");
+if ($code != 220) die("Error code $code\n");
+strtok(" \t"); //article number
+$lines = nntp_readlines($socket);
+nntp_article_store($lines);
+
+function writelog($line) {
+	print($line."\n");
+}
--- a/server.php	Tue Apr 12 00:29:41 2011 +0200
+++ b/server.php	Tue Apr 12 01:47:09 2011 +0200
@@ -1,8 +1,7 @@
 #!/usr/bin/php
 <?php
 chdir(__DIR__);
-require_once './pdo.php';
-require_once './config.php';
+require_once './common.php';
 
 $logfile = fopen('./server.log', 'w');
 $currentgroup = NULL;
@@ -25,10 +24,10 @@
 while (TRUE) {
 	$line = nntp_readline(STDIN);
 	if ($line === FALSE || $line === NULL) break;
-	$cmd = strtok($line, " \t");
-	switch (strtoupper($cmd)) {
+	$cmd = strtoupper(strtok($line, " \t"));
+	switch ($cmd) {
 		case 'LIST':
-			nntp_writeline(STDOUT, '215 List of groups follows');
+			nntp_writeline(STDOUT, '215 list of groups follows');
 			foreach ($db->evalAllAssoc('SELECT * FROM `groups`') as $group) {
 				$groupmessages = $db->evalRow('SELECT MIN(`number`), MAX(`number`) FROM `groupmessages` WHERE `group` = ?', $group['id']);
 				nntp_writeline(STDOUT, $group['name'].' '.intval($groupmessages[1]).' '.intval($groupmessages[0]).' n');
@@ -53,57 +52,35 @@
 			}
 			break;
 		case 'STAT':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '223 '.$article['messagenumber'].' <'.$article['messageid'].'> stat');
-			break;
 		case 'HEAD':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '221 '.$article['messagenumber'].' <'.$article['messageid'].'> head');
-			foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '.');
-			break;
 		case 'BODY':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '222 '.$article['messagenumber'].' <'.$article['messageid'].'> body');
-			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '.');
-			break;
 		case 'ARTICLE':
 			$article = nntp_get_article(strtok(" \t"));
 			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '220 '.$article['messagenumber'].' <'.$article['messageid'].'> article');
-			foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '');
-			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
+			switch ($cmd) {
+				case 'STAT': nntp_writeline(STDOUT, '223 '.$article['messagenumber'].' <'.$article['messageid'].'> stat'); break;
+				case 'HEAD': nntp_writeline(STDOUT, '221 '.$article['messagenumber'].' <'.$article['messageid'].'> head'); break;
+				case 'BODY': nntp_writeline(STDOUT, '222 '.$article['messagenumber'].' <'.$article['messageid'].'> body'); break;
+				case 'ARTICLE': nntp_writeline(STDOUT, '220 '.$article['messagenumber'].' <'.$article['messageid'].'> article'); break;
+				default: throw new Exception('Internal error');
+			}
+			if ($cmd == 'HEAD' || $cmd == 'ARTICLE') foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
+			if ($cmd == 'ARTICLE') nntp_writeline(STDOUT, '');
+			if ($cmd == 'BODY' || $cmd == 'ARTICLE') foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
 			nntp_writeline(STDOUT, '.');
 			break;
 		case 'LAST':
-			if ($currentarticle === NULL) {
-				nntp_writeline(STDOUT, '420 no current article has been selected');
-				break;
-			}
-			$article = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` < ? ORDER BY `number` DESC LIMIT 1', array($currentarticle['group'], $currentarticle['number']));
-			if ($article === FALSE) {
-				nntp_writeline(STDOUT, '422 no previous article in this group');
-			} else {
-				$articlea = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `id` = ?', $article['message']);
-				if ($articlea === FALSE) {
-					nntp_writeline(STDOUT, '430 no such article found');
-					return NULL;
-				}
-				$currentarticle = $article;
-				nntp_writeline(STDOUT, '223 '.$article['number'].' <'.$articlea['messageid'].'> ok');
-			}
-			break;
 		case 'NEXT':
 			if ($currentarticle === NULL) {
 				nntp_writeline(STDOUT, '420 no current article has been selected');
 				break;
 			}
-			$article = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` > ? ORDER BY `number` ASC LIMIT 1', array($currentarticle['group'], $currentarticle['number']));
+			switch ($cmd) {
+				case 'LAST': $query = 'SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` < ? ORDER BY `number` DESC LIMIT 1'; break;
+				case 'NEXT': $query = 'SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` > ? ORDER BY `number` ASC LIMIT 1'; break;
+				default: throw new Exception('Internal error');
+			}
+			$article = $db->evalRowAssoc($query, array($currentarticle['group'], $currentarticle['number']));
 			if ($article === FALSE) {
 				nntp_writeline(STDOUT, '422 no previous article in this group');
 			} else {
@@ -119,71 +96,16 @@
 		case 'POST':
 			nntp_writeline(STDOUT, '340 Ok, recommended message-ID <'.md5(time().rand()).'@pNews.Core.UCIS.nl>');
 			$lines = nntp_readlines(STDIN);
-			$header = array();
-			$headers = array();
-			while (count($lines)) {
-				$line = array_shift($lines);
-				if (!strlen($line)) break;
-				$parts = explode(': ', $line, 2);
-				$headers[strtoupper($parts[0])] = $parts[1];
-				switch (strtoupper($parts[0])) {
-					case 'PATH':
-						$line = $parts[0].': pNews.Core.UCIS.nl!'.$parts[1];
-						break;
-					case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION':
-					case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING':
-					case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO':
-					case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
-					case 'IN-REPLY-TO':
-						break;
-					case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
-					case 'NNTP-POSTING-DATE':
-						$line = NULL;
-						break;
-					default:
-						writelog("Received unknown header $parts[0]");
-				}
-				if ($line !== NULL) $header[] = $line;
+			try {
+				$msgid = nntp_article_store($lines);
+				nntp_writeline(STDOUT, '240 Article received <'.$msgid.'>');
+			} catch (Exception $ex) {
+				nntp_writeline(STDOUT, '441 '.$ex->getMessage());
 			}
-			if (!isset($headers['NEWSGROUPS'])) {
-				nntp_writeline(STDOUT, '441 Missing required Newsgroups: header');
-				break;
-			}
-			if (!isset($headers['MESSAGE-ID'])) {
-				$headers['MESSAGE-ID'] = '<'.md5(time().rand()).'@pNews.Core.UCIS.nl>';
-				$header[] = 'Message-ID: '.$headers['MESSAGE-ID'];
-			}
-			if (!isset($headers['PATH'])) {
-				$headers['PATH'] = 'pNews.Core.UCIS.nl';
-				$header[] = 'Path: '.$headers['PATH'];
-			}
-			$msgid = $headers['MESSAGE-ID'];
-			if (strlen($msgid) <= 2 || $msgid[0] != '<' || $msgid[strlen($msgid)-1] != '>') {
-				nntp_writeline(STDOUT, '441 435 Bad Message-ID');
-				break;
-			}
-			$msgid = substr($msgid, 1, -1);
-			$article = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $msgid);
-			if ($article !== FALSE) {
-				nntp_writeline(STDOUT, '441 435 Duplicate');
-				return NULL;
-			}
-			$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($msgid, implode("\r\n", $header), implode("\r\n", $lines)));
-			foreach (explode(',', $headers['NEWSGROUPS']) as $groupname) {
-				$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `name` = ?', $groupname);
-				if ($group === FALSE) continue;
-				$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
-			}
-			nntp_writeline(STDOUT, '240 Article received <'.$msgid.'>');
 			break;
 		case 'QUIT':
 			nntp_writeline(STDOUT, '205 .');
 			return;
-		case 'XOVER':
-		case 'MODE':
-		case 'CAPABILITIES':
-			nntp_writeline(STDOUT, '500 Command not implemented');
-			break;
 		default:
 			nntp_writeline(STDOUT, '500 Command not understood');
 			break;
@@ -239,26 +161,3 @@
 	global $logfile;
 	fwrite($logfile, $line."\n");
 }
-function nntp_readline($socket) {
-	global $logfile;
-	$line = fgets($socket, 512);
-	if ($line === FALSE || $line === NULL) return $line;
-	$line = rtrim($line, "\r\n");
-	fwrite($logfile, 'R: '.$line."\n");
-	return $line;
-}
-function nntp_writeline($socket, $line) {
-	global $logfile;
-	fwrite($logfile, 'W: '.$line."\n");
-	fwrite($socket, $line."\r\n");
-}
-function nntp_readlines($socket) {
-	$line = nntp_readline($socket);
-	$lines = array();
-	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
-		$lines[] = $line;
-		$line = nntp_readline($socket);
-	}
-	if ($line != '.') die("Unexpected end of message header\n");
-	return $lines;
-}