Mercurial > hg > pnewss
annotate fetchnews.php @ 1:61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
| author | Ivo Smits <Ivo@UCIS.nl> |
|---|---|
| date | Mon, 11 Apr 2011 23:17:27 +0200 |
| parents | d7ab68b71c74 |
| children | 40e545510a57 |
| rev | line source |
|---|---|
| 0 | 1 <?php |
| 2 require_once './pdo.php'; | |
| 3 require_once './config.php'; | |
| 4 | |
| 5 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) { | |
| 6 $socket = stream_socket_client($peer['address']); | |
| 7 if ($socket === FALSE) die("Could not connect to peer $peer[address]\n"); | |
| 8 $line = nntp_readline($socket); | |
| 9 $code = strtok($line, " \t"); | |
| 10 if ($code != 200) die("Error code $code from $peer[address]\n"); | |
| 11 foreach ($db->evalAllAssoc('SELECT * FROM `peergroups` WHERE `peer` = ?', $peer['id']) as $peergroup) { | |
| 12 $group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `id` = ?', $peergroup['group']); | |
| 13 nntp_writeline($socket, 'GROUP '.$group['name']); | |
| 14 $line = nntp_readline($socket); | |
| 15 $code = strtok($line, " \t"); | |
| 16 if ($code != 211) die("Error code $code from $peer[address]\n"); | |
| 17 strtok(" \t"); //number of articles | |
| 18 $low = strtok(" \t"); | |
| 19 $high = strtok(" \t"); | |
| 20 strtok(" \t"); //group name | |
| 21 if ($low != $peergroup['low'] || $high != $peergroup['high'] || $peergroup['low'] === NULL || $peergroup['high'] === NULL) { | |
| 22 for ($i = $low; $i <= $high; $i++) { | |
| 23 if ($i >= $peergroup['low'] && $i <= $peergroup['high'] && $peergroup['low'] !== NULL && $peergroup['high'] !== NULL) continue; | |
| 24 nntp_writeline($socket, 'STAT '.$i); | |
| 25 $line = nntp_readline($socket); | |
| 26 $code = strtok($line, " \t"); | |
| 27 if ($code == 423) { | |
| 28 print("Gap in article numbering at $i\n"); | |
| 29 continue; | |
| 30 } | |
| 31 if ($code != 223) die("Error code $code from $peer[address]\n"); | |
| 32 strtok(" \t"); //article number | |
| 33 $messageid = strtok(" \t"); | |
| 34 if ($messageid[0] != '<' || $messageid[strlen($messageid)-1] != '>') die("Malformed message ID $messageid\n"); | |
| 35 $messageid = substr($messageid, 1, -1); | |
| 36 $message = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid); | |
| 37 if ($message) { | |
| 38 $groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id'])); | |
| 39 if (!$groupmessage) { | |
| 40 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id'])); | |
| 41 } | |
| 42 } else { | |
| 43 nntp_writeline($socket, 'HEAD '.$i); | |
| 44 $line = nntp_readline($socket); | |
| 45 $code = strtok($line, " \t"); | |
| 46 if ($code != 221) die("Error code $code from $peer[address]\n"); | |
| 47 strtok(" \t"); //article number | |
| 48 $lines = nntp_readlines($socket); | |
|
1
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
49 nntp_removeheader(&$lines, 'Xref'); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
50 nntp_updatepath(&$lines, 'pNewss.Core.UCIS.nl'); |
| 0 | 51 $header = implode("\r\n", $lines); |
| 52 | |
| 53 nntp_writeline($socket, 'BODY '.$i); | |
| 54 $line = nntp_readline($socket); | |
| 55 $code = strtok($line, " \t"); | |
| 56 if ($code != 222) die("Error code $code from $peer[address]\n"); | |
| 57 strtok(" \t"); //article number | |
| 58 $lines = nntp_readlines($socket); | |
| 59 $body = implode("\r\n", $lines); | |
| 60 | |
| 61 $id = $db->insert('INSERT INTO `messages` (`messageid`, `header`, `body`) VALUES (?, ?, ?)', array($messageid, $header, $body)); | |
| 62 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $id)); | |
| 63 } | |
| 64 } | |
| 65 $db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group'])); | |
| 66 } | |
| 67 } | |
| 68 nntp_writeline($socket, 'QUIT'); | |
| 69 fclose($socket); | |
| 70 } | |
| 71 | |
|
1
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
72 function nntp_removeheader(&$lines, $header) { |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
73 $header = strtoupper($header).':'; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
74 $hlen = strlen($header); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
75 foreach ($lines as $key => $line) if (strtoupper(substr($line, 0, $hlen)) == $header) unset($lines[$key]); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
76 } |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
77 function nntp_updatepath(&$lines, $value) { |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
78 $header = strtoupper('Path').':'; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
79 $hlen = strlen($header); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
80 $found = FALSE; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
81 foreach ($lines as &$line) if (strtoupper(substr($line, 0, $hlen)) == $header) { |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
82 $parts = explode(': ', $line, 2); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
83 $parts[1] = $value.'!'.$parts[1]; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
84 $line = implode(': ', $parts); |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
85 $found = TRUE; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
86 break; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
87 } |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
88 if (!$found) $lines[] = 'Path: '.$value; |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
89 } |
|
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
90 |
| 0 | 91 function nntp_readline($socket) { |
| 92 $line = rtrim(fgets($socket, 512), "\r\n"); | |
| 93 print('R: '.$line."\n"); | |
| 94 return $line; | |
| 95 } | |
| 96 function nntp_writeline($socket, $line) { | |
| 97 print('W: '.$line."\n"); | |
| 98 fwrite($socket, $line."\r\n"); | |
| 99 } | |
| 100 function nntp_readlines($socket) { | |
| 101 $line = nntp_readline($socket); | |
| 102 $lines = array(); | |
| 103 while ($line != '.' && $line !== FALSE && $line !== FALSE) { | |
| 104 $lines[] = $line; | |
| 105 $line = nntp_readline($socket); | |
| 106 } | |
| 107 if ($line != '.') die("Unexpected end of message header\n"); | |
| 108 return $lines; | |
| 109 } |