changeset 11:e0807e0b1a67

Added common.php, updated server synchronization to continue after a connection failure
author Ivo Smits <Ivo@UCIS.nl>
date Sat, 18 Jun 2011 15:59:11 +0200
parents ca88deec5352
children 7917bd536187
files common.php fetchnews.php todo.txt
diffstat 3 files changed, 122 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common.php	Sat Jun 18 15:59:11 2011 +0200
@@ -0,0 +1,117 @@
+<?php
+/* Copyright 2010 Ivo Smits <Ivo@UCIS.nl>. All rights reserved.
+   Redistribution and use in source and binary forms, with or without modification, are
+   permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this list of
+      conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice, this list
+      of conditions and the following disclaimer in the documentation and/or other materials
+      provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+   WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+   FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+   ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+   ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   The views and conclusions contained in the software and documentation are those of the
+   authors and should not be interpreted as representing official policies, either expressed
+   or implied, of Ivo Smits.*/
+
+require_once './pdo.php';
+require_once './config.php';
+
+function nntp_readline($socket) {
+	$line = fgets($socket, 512);
+	if ($line === FALSE || $line === NULL) return $line;
+	$line = rtrim($line, "\r\n");
+	writelog('R: '.$line);
+	return $line;
+}
+function nntp_writeline_data($socket, $line) {
+	if (strlen($line) && $line[0] == '.') $line = '.'.$line;
+	nntp_writeline($socket, $line);
+}
+function nntp_writeline($socket, $line) {
+	writelog('W: '.$line);
+	fwrite($socket, $line."\r\n");
+}
+function nntp_readlines($socket) {
+	$line = nntp_readline($socket);
+	$lines = array();
+	while ($line != '.' && $line !== FALSE && $line !== FALSE) {
+		if (strlen($line) && $line[0] == '.') $line = substr($line, 1);
+		$lines[] = $line;
+		$line = nntp_readline($socket);
+	}
+	if ($line != '.') throw new Exception('Unexpected end of message');
+	return $lines;
+}
+
+function nntp_article_store($lines, $header = array()) {
+	global $db;
+	$headers = array();
+	if (!count($header)) {
+		while (count($lines)) {
+			$line = array_shift($lines);
+			if (!strlen($line)) break;
+			$header[] = $line;
+		}
+	}
+	foreach ($header as $headerid => $line) {
+		if (!strlen($line) || $line == '.') throw new Exception('Empty or terminating header line');
+		if (strpos($line, "\r") !== FALSE || strpos($line, "\n") !== FALSE || strpos($line, "\0")) throw new Exception('Invalid newline or NUL character in header line');
+		$parts = explode(': ', $line, 2);
+		$headername = strtoupper($parts[0]);
+		switch ($headername) {
+			case 'PATH': case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'MESSAGE-ID': case 'SENDER':
+				if (isset($headers[$headername])) throw new Exception('Duplicate header: '.$headername);
+				$headers[strtoupper($parts[0])] = $parts[1];
+				unset($header[$headerid]);
+				break;
+			case 'ORGANIZATION': case 'LINES':
+			case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING': case 'USER-AGENT':
+			case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO': case 'IN-REPLY-TO':
+			case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY':
+				break;
+			case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO':
+			case 'NNTP-POSTING-DATE':
+				unset($header[$headerid]);
+				$line = NULL;
+				break;
+			default:
+				writelog("Received unknown header $headername");
+		}
+	}
+	foreach ($lines as $line) {
+		if ($line == '.') throw new Exception('Terminating body line');
+		if (strpos($line, "\r") !== FALSE || strpos($line, "\n") !== FALSE || strpos($line, "\0")) throw new Exception('Invalid newline or NUL character in header line');
+	}
+	if (!isset($headers['NEWSGROUPS'])) throw new Exception('Missing required Newsgroups: header');
+	$newsgroups = array();
+	foreach (explode(',', $headers['NEWSGROUPS']) as $groupname) {
+		$group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `name` = ?', $groupname);
+		if ($group === FALSE) continue;
+		$newsgroups[] = $group['id'];
+	}
+	if (!count($newsgroups)) throw new Exception('No known newsgroups listed');
+	if (!isset($headers['MESSAGE-ID'])) $headers['MESSAGE-ID'] = '<'.md5(time().rand()).'@pNewss.Core.UCIS.nl>';
+	$messageid = $headers['MESSAGE-ID'];
+	if (strlen($messageid) < 3 || strlen($messageid) > 250 || $messageid[0] != '<' || strpos($messageid, '>') !== strlen($messageid) - 1) throw new Exception('Bad Message-ID');
+	$messageid = substr($messageid, 1, -1);
+	$article = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid);
+	if ($article !== FALSE) throw new Exception('Duplicate');
+	$headers['PATH'] = 'pNewss.Core.UCIS.nl'.(isset($headers['PATH'])?'!'.$headers['PATH']:'');
+	foreach (array('Path', 'From', 'Newsgroups', 'Subject', 'Date', 'Message-ID', 'Sender') as $headername) {
+		if (isset($headers[strtoupper($headername)])) $header[] = $headername.': '.$headers[strtoupper($headername)];
+	}
+	$id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, implode("\r\n", $header), implode("\r\n", $lines)));
+	foreach ($newsgroups as $groupid) $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($groupid, $id));
+	return $messageid;
+}
--- a/fetchnews.php	Thu Apr 21 00:01:32 2011 +0200
+++ b/fetchnews.php	Sat Jun 18 15:59:11 2011 +0200
@@ -30,7 +30,10 @@
 
 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) {
 	$socket = stream_socket_client($peer['address']);
-	if ($socket === FALSE) die("Could not connect to peer $peer[address]\n");
+	if ($socket === FALSE) {
+		print("Could not connect to peer $peer[address]\n");
+		continue;
+	}
 	$line = nntp_readline($socket);
 	$code = strtok($line, " \t");
 	if ($code == 200) {
--- a/todo.txt	Thu Apr 21 00:01:32 2011 +0200
+++ b/todo.txt	Sat Jun 18 15:59:11 2011 +0200
@@ -2,3 +2,4 @@
 - Allow to store (part of) article data in file
 - Expire articles after a certain time
 - Reject old articles
+- Simpler configuration (CLI, configuration file, ...)