# HG changeset patch
# User Ivo Smits <Ivo@UCIS.nl>
# Date 1302565629 -7200
# Node ID 0dcdb73cbcbf7b8ddc874f5d9e33db8425d8db79
# Parent  40e545510a57e891249d318feab2964d9de1728c
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article

diff -r 40e545510a57 -r 0dcdb73cbcbf database.mysql
--- a/database.mysql	Tue Apr 12 00:29:41 2011 +0200
+++ b/database.mysql	Tue Apr 12 01:47:09 2011 +0200
@@ -3,7 +3,7 @@
 -- http://www.phpmyadmin.net
 --
 -- Machine: localhost
--- Genereertijd: 12 Apr 2011 om 00:18
+-- Genereertijd: 12 Apr 2011 om 01:45
 -- Serverversie: 5.0.51
 -- PHP-Versie: 5.3.3-7
 
@@ -50,7 +50,7 @@
   `id` int(10) unsigned NOT NULL auto_increment,
   `messageid` varchar(255) character set ascii NOT NULL,
   `header` text character set ascii NOT NULL,
-  `body` text character set ascii NOT NULL,
+  `body` mediumtext character set ascii NOT NULL,
   PRIMARY KEY  (`id`),
   UNIQUE KEY `messageid` (`messageid`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
diff -r 40e545510a57 -r 0dcdb73cbcbf fetchnews.php
--- a/fetchnews.php	Tue Apr 12 00:29:41 2011 +0200
+++ b/fetchnews.php	Tue Apr 12 01:47:09 2011 +0200
@@ -1,7 +1,7 @@
+#!/usr/bin/php
 <?php
 chdir(__DIR__);
-require_once './pdo.php';
-require_once './config.php';
+require_once './common.php';
 
 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) {
 	$socket = stream_socket_client($peer['address']);
@@ -44,26 +44,17 @@
 						$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id']));
 					}
 				} else {
-					nntp_writeline($socket, 'HEAD '.$i);
+					nntp_writeline($socket, 'ARTICLE '.$i);
 					$line = nntp_readline($socket);
 					$code = strtok($line, " \t");
-					if ($code != 221) die("Error code $code from $peer[address]\n");
+					if ($code != 220) die("Error code $code from $peer[address]\n");
 					strtok(" \t"); //article number
 					$lines = nntp_readlines($socket);
-					nntp_removeheader(&$lines, 'Xref');
-					nntp_updatepath(&$lines, 'pNewss.Core.UCIS.nl');
-					$header = implode("\r\n", $lines);
-					
-					nntp_writeline($socket, 'BODY '.$i);
-					$line = nntp_readline($socket);
-					$code = strtok($line, " \t");
-					if ($code != 222) die("Error code $code from $peer[address]\n");
-					strtok(" \t"); //article number
-					$lines = nntp_readlines($socket);
-					$body = implode("\r\n", $lines);
-					
-					$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, $header, $body));
-					$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
+					try {
+						nntp_article_store($lines);
+					} catch (Exception $ex) {
+						writelog($ex->getMessage());
+					}
 				}
 			}
 			$db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group']));
@@ -113,41 +104,6 @@
 	fclose($socket);
 }
 
-function nntp_removeheader(&$lines, $header) {
-	$header = strtoupper($header).':';
-	$hlen = strlen($header);
-	foreach ($lines as $key => $line) if (strtoupper(substr($line, 0, $hlen)) == $header) unset($lines[$key]);
-}
-function nntp_updatepath(&$lines, $value) {
-	$header = strtoupper('Path').':';
-	$hlen = strlen($header);
-	$found = FALSE;
-	foreach ($lines as &$line) if (strtoupper(substr($line, 0, $hlen)) == $header) {
-		$parts = explode(': ', $line, 2);
-		$parts[1] = $value.'!'.$parts[1];
-		$line = implode(': ', $parts);
-		$found = TRUE;
-		break;
-	}
-	if (!$found) $lines[] = 'Path: '.$value;
+function writelog($line) {
+	print($line."\n");
 }
-
-function nntp_readline($socket) {
-	$line = rtrim(fgets($socket, 512), "\r\n");
-	print('R: '.$line."\n");
-	return $line;
-}
-function nntp_writeline($socket, $line) {
-	print('W: '.$line."\n");
-	fwrite($socket, $line."\r\n");
-}
-function nntp_readlines($socket) {
-	$line = nntp_readline($socket);
-	$lines = array();
-	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
-		$lines[] = $line;
-		$line = nntp_readline($socket);
-	}
-	if ($line != '.') die("Unexpected end of message header\n");
-	return $lines;
-}
diff -r 40e545510a57 -r 0dcdb73cbcbf reloadarticle.php
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/reloadarticle.php	Tue Apr 12 01:47:09 2011 +0200
@@ -0,0 +1,26 @@
+#!/usr/bin/php
+<?php
+chdir(__DIR__);
+require_once './common.php';
+
+if (count($_SERVER['argv']) < 2) die('Not enough arguments (usage: [server] [messageid])');
+$messageid = $_SERVER['argv'][2];
+$db->update('DELETE FROM `messages` WHERE `messageid` = ?', $messageid);
+$socket = stream_socket_client($_SERVER['argv'][1]);
+if ($socket === FALSE) die("Could not connect to peer\n");
+$line = nntp_readline($socket);
+$code = strtok($line, " \t");
+if ($code == 200) {
+} else if ($code == 201) {
+} else die("Error code $code\n");
+nntp_writeline($socket, 'ARTICLE <'.$messageid.'>');
+$line = nntp_readline($socket);
+$code = strtok($line, " \t");
+if ($code != 220) die("Error code $code\n");
+strtok(" \t"); //article number
+$lines = nntp_readlines($socket);
+nntp_article_store($lines);
+
+function writelog($line) {
+	print($line."\n");
+}
diff -r 40e545510a57 -r 0dcdb73cbcbf server.php
--- a/server.php	Tue Apr 12 00:29:41 2011 +0200
+++ b/server.php	Tue Apr 12 01:47:09 2011 +0200
@@ -1,8 +1,7 @@
 #!/usr/bin/php
 <?php
 chdir(__DIR__);
-require_once './pdo.php';
-require_once './config.php';
+require_once './common.php';
 
 $logfile = fopen('./server.log', 'w');
 $currentgroup = NULL;
@@ -25,10 +24,10 @@
 while (TRUE) {
 	$line = nntp_readline(STDIN);
 	if ($line === FALSE || $line === NULL) break;
-	$cmd = strtok($line, " \t");
-	switch (strtoupper($cmd)) {
+	$cmd = strtoupper(strtok($line, " \t"));
+	switch ($cmd) {
 		case 'LIST':
-			nntp_writeline(STDOUT, '215 List of groups follows');
+			nntp_writeline(STDOUT, '215 list of groups follows');
 			foreach ($db->evalAllAssoc('SELECT * FROM `groups`') as $group) {
 				$groupmessages = $db->evalRow('SELECT MIN(`number`), MAX(`number`) FROM `groupmessages` WHERE `group` = ?', $group['id']);
 				nntp_writeline(STDOUT, $group['name'].' '.intval($groupmessages[1]).' '.intval($groupmessages[0]).' n');
@@ -53,57 +52,35 @@
 			}
 			break;
 		case 'STAT':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '223 '.$article['messagenumber'].' <'.$article['messageid'].'> stat');
-			break;
 		case 'HEAD':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '221 '.$article['messagenumber'].' <'.$article['messageid'].'> head');
-			foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '.');
-			break;
 		case 'BODY':
-			$article = nntp_get_article(strtok(" \t"));
-			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '222 '.$article['messagenumber'].' <'.$article['messageid'].'> body');
-			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '.');
-			break;
 		case 'ARTICLE':
 			$article = nntp_get_article(strtok(" \t"));
 			if ($article === NULL) break;
-			nntp_writeline(STDOUT, '220 '.$article['messagenumber'].' <'.$article['messageid'].'> article');
-			foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
-			nntp_writeline(STDOUT, '');
-			foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
+			switch ($cmd) {
+				case 'STAT': nntp_writeline(STDOUT, '223 '.$article['messagenumber'].' <'.$article['messageid'].'> stat'); break;
+				case 'HEAD': nntp_writeline(STDOUT, '221 '.$article['messagenumber'].' <'.$article['messageid'].'> head'); break;
+				case 'BODY': nntp_writeline(STDOUT, '222 '.$article['messagenumber'].' <'.$article['messageid'].'> body'); break;
+				case 'ARTICLE': nntp_writeline(STDOUT, '220 '.$article['messagenumber'].' <'.$article['messageid'].'> article'); break;
+				default: throw new Exception('Internal error');
+			}
+			if ($cmd == 'HEAD' || $cmd == 'ARTICLE') foreach (explode("\r\n", $article['header']) as $line) nntp_writeline(STDOUT, $line);
+			if ($cmd == 'ARTICLE') nntp_writeline(STDOUT, '');
+			if ($cmd == 'BODY' || $cmd == 'ARTICLE') foreach (explode("\r\n", $article['body']) as $line) nntp_writeline(STDOUT, $line);
 			nntp_writeline(STDOUT, '.');
 			break;
 		case 'LAST':
-			if ($currentarticle === NULL) {
-				nntp_writeline(STDOUT, '420 no current article has been selected');
-				break;
-			}
-			$article = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` < ? ORDER BY `number` DESC LIMIT 1', array($currentarticle['group'], $currentarticle['number']));
-			if ($article === FALSE) {
-				nntp_writeline(STDOUT, '422 no previous article in this group');
-			} else {
-				$articlea = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `id` = ?', $article['message']);
-				if ($articlea === FALSE) {
-					nntp_writeline(STDOUT, '430 no such article found');
-					return NULL;
-				}
-				$currentarticle = $article;
-				nntp_writeline(STDOUT, '223 '.$article['number'].' <'.$articlea['messageid'].'> ok');
-			}
-			break;
 		case 'NEXT':
 			if ($currentarticle === NULL) {
 				nntp_writeline(STDOUT, '420 no current article has been selected');
 				break;
 			}
-			$article = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` > ? ORDER BY `number` ASC LIMIT 1', array($currentarticle['group'], $currentarticle['number']));
+			switch ($cmd) {
+				case 'LAST': $query = 'SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` < ? ORDER BY `number` DESC LIMIT 1'; break;
+				case 'NEXT': $query = 'SELECT * FROM `groupmessages` WHERE `group` = ? AND `number` > ? ORDER BY `number` ASC LIMIT 1'; break;
+				default: throw new Exception('Internal error');
+			}
+			$article = $db->evalRowAssoc($query, array($currentarticle['group'], $currentarticle['number']));
 			if ($article === FALSE) {
 				nntp_writeline(STDOUT, '422 no previous article in this group');
 			} else {
@@ -119,71 +96,16 @@
 		case 'POST':
 			nntp_writeline(STDOUT, '340 Ok, recommended message-ID <'.md5(time().rand()).'@pNews.Core.UCIS.nl>');
 			$lines = nntp_readlines(STDIN);
-			$header = array();
-			$headers = array();
-			while (count($lines)) {
-				$line = array_shift($lines);
-				if (!strlen($line)) break;
-				$parts = explode(': ', $line, 2);
-				$headers[strtoupper($parts[0])] = $parts[1];
-				switch (strtoupper($parts[0])) {
-					case 'PATH':
-						$line = $parts[0].': pNews.Core.UCIS.nl!'.$parts[1];
-						break;
-					case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION':
-					case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING':
-					case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO':
-					case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
-					case 'IN-REPLY-TO':
-						break;
-					case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
-					case 'NNTP-POSTING-DATE':
-						$line = NULL;
-						break;
-					default:
-						writelog("Received unknown header $parts[0]");
-				}
-				if ($line !== NULL) $header[] = $line;
+			try {
+				$msgid = nntp_article_store($lines);
+				nntp_writeline(STDOUT, '240 Article received <'.$msgid.'>');
+			} catch (Exception $ex) {
+				nntp_writeline(STDOUT, '441 '.$ex->getMessage());
 			}
-			if (!isset($headers['NEWSGROUPS'])) {
-				nntp_writeline(STDOUT, '441 Missing required Newsgroups: header');
-				break;
-			}
-			if (!isset($headers['MESSAGE-ID'])) {
-				$headers['MESSAGE-ID'] = '<'.md5(time().rand()).'@pNews.Core.UCIS.nl>';
-				$header[] = 'Message-ID: '.$headers['MESSAGE-ID'];
-			}
-			if (!isset($headers['PATH'])) {
-				$headers['PATH'] = 'pNews.Core.UCIS.nl';
-				$header[] = 'Path: '.$headers['PATH'];
-			}
-			$msgid = $headers['MESSAGE-ID'];
-			if (strlen($msgid) <= 2 || $msgid[0] != '<' || $msgid[strlen($msgid)-1] != '>') {
-				nntp_writeline(STDOUT, '441 435 Bad Message-ID');
-				break;
-			}
-			$msgid = substr($msgid, 1, -1);
-			$article = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $msgid);
-			if ($article !== FALSE) {
-				nntp_writeline(STDOUT, '441 435 Duplicate');
-				return NULL;
-			}
-			$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($msgid, implode("\r\n", $header), implode("\r\n", $lines)));
-			foreach (explode(',', $headers['NEWSGROUPS']) as $groupname) {
-				$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `name` = ?', $groupname);
-				if ($group === FALSE) continue;
-				$db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
-			}
-			nntp_writeline(STDOUT, '240 Article received <'.$msgid.'>');
 			break;
 		case 'QUIT':
 			nntp_writeline(STDOUT, '205 .');
 			return;
-		case 'XOVER':
-		case 'MODE':
-		case 'CAPABILITIES':
-			nntp_writeline(STDOUT, '500 Command not implemented');
-			break;
 		default:
 			nntp_writeline(STDOUT, '500 Command not understood');
 			break;
@@ -239,26 +161,3 @@
 	global $logfile;
 	fwrite($logfile, $line."\n");
 }
-function nntp_readline($socket) {
-	global $logfile;
-	$line = fgets($socket, 512);
-	if ($line === FALSE || $line === NULL) return $line;
-	$line = rtrim($line, "\r\n");
-	fwrite($logfile, 'R: '.$line."\n");
-	return $line;
-}
-function nntp_writeline($socket, $line) {
-	global $logfile;
-	fwrite($logfile, 'W: '.$line."\n");
-	fwrite($socket, $line."\r\n");
-}
-function nntp_readlines($socket) {
-	$line = nntp_readline($socket);
-	$lines = array();
-	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
-		$lines[] = $line;
-		$line = nntp_readline($socket);
-	}
-	if ($line != '.') die("Unexpected end of message header\n");
-	return $lines;
-}