Mercurial > hg > pnewss
annotate fetchnews.php @ 6:bc6045ed0b2e
Added script to fix and re-index database fields and references
| author | Ivo Smits <Ivo@UCIS.nl> |
|---|---|
| date | Tue, 12 Apr 2011 11:41:35 +0200 |
| parents | 0dcdb73cbcbf |
| children | 01dc7eeaf5df |
| rev | line source |
|---|---|
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
1 #!/usr/bin/php |
| 0 | 2 <?php |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
3 chdir(__DIR__); |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
4 require_once './common.php'; |
| 0 | 5 |
| 6 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) { | |
| 7 $socket = stream_socket_client($peer['address']); | |
| 8 if ($socket === FALSE) die("Could not connect to peer $peer[address]\n"); | |
| 9 $line = nntp_readline($socket); | |
| 10 $code = strtok($line, " \t"); | |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
11 if ($code == 200) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
12 } else if ($code == 201) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
13 $peer['post'] = 0; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
14 } else die("Error code $code from $peer[address]\n"); |
| 0 | 15 foreach ($db->evalAllAssoc('SELECT * FROM `peergroups` WHERE `peer` = ?', $peer['id']) as $peergroup) { |
| 16 $group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `id` = ?', $peergroup['group']); | |
| 17 nntp_writeline($socket, 'GROUP '.$group['name']); | |
| 18 $line = nntp_readline($socket); | |
| 19 $code = strtok($line, " \t"); | |
| 20 if ($code != 211) die("Error code $code from $peer[address]\n"); | |
| 21 strtok(" \t"); //number of articles | |
| 22 $low = strtok(" \t"); | |
| 23 $high = strtok(" \t"); | |
| 24 strtok(" \t"); //group name | |
| 25 if ($low != $peergroup['low'] || $high != $peergroup['high'] || $peergroup['low'] === NULL || $peergroup['high'] === NULL) { | |
| 26 for ($i = $low; $i <= $high; $i++) { | |
| 27 if ($i >= $peergroup['low'] && $i <= $peergroup['high'] && $peergroup['low'] !== NULL && $peergroup['high'] !== NULL) continue; | |
| 28 nntp_writeline($socket, 'STAT '.$i); | |
| 29 $line = nntp_readline($socket); | |
| 30 $code = strtok($line, " \t"); | |
| 31 if ($code == 423) { | |
| 32 print("Gap in article numbering at $i\n"); | |
| 33 continue; | |
| 34 } | |
| 35 if ($code != 223) die("Error code $code from $peer[address]\n"); | |
| 36 strtok(" \t"); //article number | |
| 37 $messageid = strtok(" \t"); | |
| 38 if ($messageid[0] != '<' || $messageid[strlen($messageid)-1] != '>') die("Malformed message ID $messageid\n"); | |
| 39 $messageid = substr($messageid, 1, -1); | |
| 40 $message = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid); | |
| 41 if ($message) { | |
| 42 $groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id'])); | |
| 43 if (!$groupmessage) { | |
| 44 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id'])); | |
| 45 } | |
| 46 } else { | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
47 nntp_writeline($socket, 'ARTICLE '.$i); |
| 0 | 48 $line = nntp_readline($socket); |
| 49 $code = strtok($line, " \t"); | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
50 if ($code != 220) die("Error code $code from $peer[address]\n"); |
| 0 | 51 strtok(" \t"); //article number |
| 52 $lines = nntp_readlines($socket); | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
53 try { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
54 nntp_article_store($lines); |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
55 } catch (Exception $ex) { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
56 writelog($ex->getMessage()); |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
57 } |
| 0 | 58 } |
| 59 } | |
| 60 $db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group'])); | |
| 61 } | |
| 62 } | |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
63 while ($peer['post']) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
64 if ($peer['lastposted'] === NULL) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
65 $articles = $db->evalAllAssoc('SELECT * FROM `messages` LIMIT 10'); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
66 } else { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
67 $articles = $db->evalAllAssoc('SELECT * FROM `messages` WHERE `id` > ? LIMIT 10', $peer['lastposted']); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
68 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
69 if (!count($articles)) break; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
70 foreach ($articles as $article) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
71 nntp_writeline($socket, 'POST'); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
72 $line = nntp_readline($socket); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
73 $code = strtok($line, " \t"); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
74 if ($code != 340) die("Error code $code from $peer[address]\n"); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
75 foreach (explode("\r\n", $article['header']) as $line) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
76 $parts = explode(': ', $line, 2); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
77 switch (strtoupper($parts[0])) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
78 case 'PATH': case 'FROM': case 'NEWSGROUPS': case 'SUBJECT': case 'DATE': case 'ORGANIZATION': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
79 case 'LINES': case 'MESSAGE-ID': case 'MIME-VERSION': case 'CONTENT-TYPE': case 'CONTENT-TRANSFER-ENCODING': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
80 case 'USER-AGENT': case 'REFERENCES': case 'REPLY-TO': case 'SENDER': case 'FOLLOWUP-TO': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
81 case 'EXPIRES': case 'CONTROL': case 'DISTRIBUTION': case 'KEYWORDS': case 'SUMMARY': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
82 case 'IN-REPLY-TO': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
83 break; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
84 case 'NNTP-POSTING-HOST': case 'X-TRACE': case 'XREF': case 'X-COMPLAINTS-TO': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
85 case 'NNTP-POSTING-DATE': |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
86 $line = NULL; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
87 break; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
88 default: |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
89 print("Sending unknown header $parts[0]\n"); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
90 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
91 if ($line !== NULL) nntp_writeline($socket, $line); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
92 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
93 nntp_writeline($socket, ''); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
94 foreach (explode("\r\n", $article['body']) as $line) nntp_writeline($socket, $line); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
95 nntp_writeline($socket, '.'); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
96 $line = nntp_readline($socket); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
97 $code = strtok($line, " \t"); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
98 if ($code != 240) print("Article $article[messageid] was not accepted ($code)\n"); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
99 if ($article['id'] > $peer['lastposted']) $peer['lastposted'] = $article['id']; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
100 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
101 $db->update('UPDATE `peers` SET `lastposted` = ? WHERE `id` = ?', array($peer['lastposted'], $peer['id'])); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
102 } |
| 0 | 103 nntp_writeline($socket, 'QUIT'); |
| 104 fclose($socket); | |
| 105 } | |
| 106 | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
107 function writelog($line) { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
108 print($line."\n"); |
|
1
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
109 } |