Mercurial > hg > pnewss

comparison fetchnews.php @ 3:0dcdb73cbcbf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
author Ivo Smits <Ivo@UCIS.nl>
date Tue, 12 Apr 2011 01:47:09 +0200
parents 40e545510a57
children 01dc7eeaf5df
comparison
equal deleted inserted replaced
2:40e545510a57 3:0dcdb73cbcbf
1 #!/usr/bin/php
1 <?php 2 <?php
2 chdir(__DIR__); 3 chdir(__DIR__);
3 require_once './pdo.php'; 4 require_once './common.php';
4 require_once './config.php';
5 5
6 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) { 6 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) {
7 $socket = stream_socket_client($peer['address']); 7 $socket = stream_socket_client($peer['address']);
8 if ($socket === FALSE) die("Could not connect to peer $peer[address]\n"); 8 if ($socket === FALSE) die("Could not connect to peer $peer[address]\n");
9 $line = nntp_readline($socket); 9 $line = nntp_readline($socket);
42 $groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id'])); 42 $groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id']));
43 if (!$groupmessage) { 43 if (!$groupmessage) {
44 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id'])); 44 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id']));
45 } 45 }
46 } else { 46 } else {
47 nntp_writeline($socket, 'HEAD '.$i); 47 nntp_writeline($socket, 'ARTICLE '.$i);
48 $line = nntp_readline($socket); 48 $line = nntp_readline($socket);
49 $code = strtok($line, " \t"); 49 $code = strtok($line, " \t");
50 if ($code != 221) die("Error code $code from $peer[address]\n"); 50 if ($code != 220) die("Error code $code from $peer[address]\n");
51 strtok(" \t"); //article number 51 strtok(" \t"); //article number
52 $lines = nntp_readlines($socket); 52 $lines = nntp_readlines($socket);
53 nntp_removeheader(&$lines, 'Xref'); 53 try {
54 nntp_updatepath(&$lines, 'pNewss.Core.UCIS.nl'); 54 nntp_article_store($lines);
55 $header = implode("\r\n", $lines); 55 } catch (Exception $ex) {
56 56 writelog($ex->getMessage());
57 nntp_writeline($socket, 'BODY '.$i); 57 }
58 $line = nntp_readline($socket);
59 $code = strtok($line, " \t");
60 if ($code != 222) die("Error code $code from $peer[address]\n");
61 strtok(" \t"); //article number
62 $lines = nntp_readlines($socket);
63 $body = implode("\r\n", $lines);
64
65 $id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, $header, $body));
66 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id));
67 } 58 }
68 } 59 }
69 $db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group'])); 60 $db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group']));
70 } 61 }
71 } 62 }
111 } 102 }
112 nntp_writeline($socket, 'QUIT'); 103 nntp_writeline($socket, 'QUIT');
113 fclose($socket); 104 fclose($socket);
114 } 105 }
115 106
116 function nntp_removeheader(&$lines, $header) { 107 function writelog($line) {
117 $header = strtoupper($header).':'; 108 print($line."\n");
118 $hlen = strlen($header);
119 foreach ($lines as $key => $line) if (strtoupper(substr($line, 0, $hlen)) == $header) unset($lines[$key]);
120 } 109 }
121 function nntp_updatepath(&$lines, $value) {
122 $header = strtoupper('Path').':';
123 $hlen = strlen($header);
124 $found = FALSE;
125 foreach ($lines as &$line) if (strtoupper(substr($line, 0, $hlen)) == $header) {
126 $parts = explode(': ', $line, 2);
127 $parts[1] = $value.'!'.$parts[1];
128 $line = implode(': ', $parts);
129 $found = TRUE;
130 break;
131 }
132 if (!$found) $lines[] = 'Path: '.$value;
133 }
134
135 function nntp_readline($socket) {
136 $line = rtrim(fgets($socket, 512), "\r\n");
137 print('R: '.$line."\n");
138 return $line;
139 }
140 function nntp_writeline($socket, $line) {
141 print('W: '.$line."\n");
142 fwrite($socket, $line."\r\n");
143 }
144 function nntp_readlines($socket) {
145 $line = nntp_readline($socket);
146 $lines = array();
147 while ($line != '.' && $line !== FALSE && $line !== FALSE) {
148 $lines[] = $line;
149 $line = nntp_readline($socket);
150 }
151 if ($line != '.') die("Unexpected end of message header\n");
152 return $lines;
153 }