# HG changeset patch # User Ivo Smits # Date 1308405551 -7200 # Node ID e0807e0b1a678d34f58ac7625e3ffc02c87ec2ae # Parent ca88deec535259bcef4c33801a6ede1bfcdad6e6 Added common.php, updated server synchronization to continue after a connection failure diff -r ca88deec5352 -r e0807e0b1a67 common.php --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/common.php Sat Jun 18 15:59:11 2011 +0200 @@ -0,0 +1,117 @@ +. All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are + permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list + of conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + The views and conclusions contained in the software and documentation are those of the + authors and should not be interpreted as representing official policies, either expressed + or implied, of Ivo Smits.*/ + +require_once './pdo.php'; +require_once './config.php'; + +function nntp_readline($socket) { + $line = fgets($socket, 512); + if ($line === FALSE || $line === NULL) return $line; + $line = rtrim($line, "\r\n"); + writelog('R: '.$line); + return $line; +} +function nntp_writeline_data($socket, $line) { + if (strlen($line) && $line[0] == '.') $line = '.'.$line; + nntp_writeline($socket, $line); +} +function nntp_writeline($socket, $line) { + writelog('W: '.$line); + fwrite($socket, $line."\r\n"); +} +function nntp_readlines($socket) { + $line = nntp_readline($socket); + $lines = array(); + while ($line != '.' && $line !== FALSE && $line !== FALSE) { + if (strlen($line) && $line[0] == '.') $line = substr($line, 1); + $lines[] = $line; + $line = nntp_readline($socket); + } + if ($line != '.') throw new Exception('Unexpected end of message'); + return $lines; +} + +function nntp_article_store($lines, $header = array()) { + global $db; + $headers = array(); + if (!count($header)) { + while (count($lines)) { + $line = array_shift($lines); + if (!strlen($line)) break; + $header[] = $line; + } + } + foreach ($header as $headerid => $line) { + if (!strlen($line) || $line == '.') throw new Exception('Empty or terminating header line'); + if (strpos($line, "\r") !== FALSE || strpos($line, "\n") !== FALSE || strpos($line, "\0")) throw new Exception('Invalid newline or NUL character in header line'); + $parts = explode(': ', $line, 2); + $headername = strtoupper($parts[0]); + switch ($headername) { + case 'PATH': case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'MESSAGE-ID': case 'SENDER': + if (isset($headers[$headername])) throw new Exception('Duplicate header: '.$headername); + $headers[strtoupper($parts[0])] = $parts[1]; + unset($header[$headerid]); + break; + case 'ORGANIZATION': case 'LINES': + case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING': case 'USER-AGENT': + case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO': case 'IN-REPLY-TO': + case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY': + break; + case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO': + case 'NNTP-POSTING-DATE': + unset($header[$headerid]); + $line = NULL; + break; + default: + writelog("Received unknown header $headername"); + } + } + foreach ($lines as $line) { + if ($line == '.') throw new Exception('Terminating body line'); + if (strpos($line, "\r") !== FALSE || strpos($line, "\n") !== FALSE || strpos($line, "\0")) throw new Exception('Invalid newline or NUL character in header line'); + } + if (!isset($headers['NEWSGROUPS'])) throw new Exception('Missing required Newsgroups: header'); + $newsgroups = array(); + foreach (explode(',', $headers['NEWSGROUPS']) as $groupname) { + $group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `name` = ?', $groupname); + if ($group === FALSE) continue; + $newsgroups[] = $group['id']; + } + if (!count($newsgroups)) throw new Exception('No known newsgroups listed'); + if (!isset($headers['MESSAGE-ID'])) $headers['MESSAGE-ID'] = '<'.md5(time().rand()).'@pNewss.Core.UCIS.nl>'; + $messageid = $headers['MESSAGE-ID']; + if (strlen($messageid) < 3 || strlen($messageid) > 250 || $messageid[0] != '<' || strpos($messageid, '>') !== strlen($messageid) - 1) throw new Exception('Bad Message-ID'); + $messageid = substr($messageid, 1, -1); + $article = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid); + if ($article !== FALSE) throw new Exception('Duplicate'); + $headers['PATH'] = 'pNewss.Core.UCIS.nl'.(isset($headers['PATH'])?'!'.$headers['PATH']:''); + foreach (array('Path', 'From', 'Newsgroups', 'Subject', 'Date', 'Message-ID', 'Sender') as $headername) { + if (isset($headers[strtoupper($headername)])) $header[] = $headername.': '.$headers[strtoupper($headername)]; + } + $id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, implode("\r\n", $header), implode("\r\n", $lines))); + foreach ($newsgroups as $groupid) $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($groupid, $id)); + return $messageid; +} diff -r ca88deec5352 -r e0807e0b1a67 fetchnews.php --- a/fetchnews.php Thu Apr 21 00:01:32 2011 +0200 +++ b/fetchnews.php Sat Jun 18 15:59:11 2011 +0200 @@ -30,7 +30,10 @@ foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) { $socket = stream_socket_client($peer['address']); - if ($socket === FALSE) die("Could not connect to peer $peer[address]\n"); + if ($socket === FALSE) { + print("Could not connect to peer $peer[address]\n"); + continue; + } $line = nntp_readline($socket); $code = strtok($line, " \t"); if ($code == 200) { diff -r ca88deec5352 -r e0807e0b1a67 todo.txt --- a/todo.txt Thu Apr 21 00:01:32 2011 +0200 +++ b/todo.txt Sat Jun 18 15:59:11 2011 +0200 @@ -2,3 +2,4 @@ - Allow to store (part of) article data in file - Expire articles after a certain time - Reject old articles +- Simpler configuration (CLI, configuration file, ...)