Mercurial > hg > pnewss
annotate fetchnews.php @ 11:e0807e0b1a67
Added common.php, updated server synchronization to continue after a connection failure
| author | Ivo Smits <Ivo@UCIS.nl> |
|---|---|
| date | Sat, 18 Jun 2011 15:59:11 +0200 |
| parents | ae0c67d72670 |
| children | 7917bd536187 |
| rev | line source |
|---|---|
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
1 #!/usr/bin/php |
| 0 | 2 <?php |
| 8 | 3 /* Copyright 2010 Ivo Smits <Ivo@UCIS.nl>. All rights reserved. |
| 4 Redistribution and use in source and binary forms, with or without modification, are | |
| 5 permitted provided that the following conditions are met: | |
| 6 | |
| 7 1. Redistributions of source code must retain the above copyright notice, this list of | |
| 8 conditions and the following disclaimer. | |
| 9 | |
| 10 2. Redistributions in binary form must reproduce the above copyright notice, this list | |
| 11 of conditions and the following disclaimer in the documentation and/or other materials | |
| 12 provided with the distribution. | |
| 13 | |
| 14 THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED | |
| 15 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND | |
| 16 FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR | |
| 17 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 18 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |
| 19 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON | |
| 20 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | |
| 21 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |
| 22 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 23 | |
| 24 The views and conclusions contained in the software and documentation are those of the | |
| 25 authors and should not be interpreted as representing official policies, either expressed | |
| 26 or implied, of Ivo Smits.*/ | |
| 27 | |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
28 chdir(__DIR__); |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
29 require_once './common.php'; |
| 0 | 30 |
| 31 foreach ($db->evalAllAssoc('SELECT * FROM `peers`') as $peer) { | |
| 32 $socket = stream_socket_client($peer['address']); | |
|
11
e0807e0b1a67
Added common.php, updated server synchronization to continue after a connection failure
Ivo Smits <Ivo@UCIS.nl>
parents:
9
diff
changeset
|
33 if ($socket === FALSE) { |
|
e0807e0b1a67
Added common.php, updated server synchronization to continue after a connection failure
Ivo Smits <Ivo@UCIS.nl>
parents:
9
diff
changeset
|
34 print("Could not connect to peer $peer[address]\n"); |
|
e0807e0b1a67
Added common.php, updated server synchronization to continue after a connection failure
Ivo Smits <Ivo@UCIS.nl>
parents:
9
diff
changeset
|
35 continue; |
|
e0807e0b1a67
Added common.php, updated server synchronization to continue after a connection failure
Ivo Smits <Ivo@UCIS.nl>
parents:
9
diff
changeset
|
36 } |
| 0 | 37 $line = nntp_readline($socket); |
| 38 $code = strtok($line, " \t"); | |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
39 if ($code == 200) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
40 } else if ($code == 201) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
41 $peer['post'] = 0; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
42 } else die("Error code $code from $peer[address]\n"); |
| 0 | 43 foreach ($db->evalAllAssoc('SELECT * FROM `peergroups` WHERE `peer` = ?', $peer['id']) as $peergroup) { |
| 44 $group = $db->evalRowAssoc('SELECT * FROM `groups` WHERE `id` = ?', $peergroup['group']); | |
| 45 nntp_writeline($socket, 'GROUP '.$group['name']); | |
| 46 $line = nntp_readline($socket); | |
| 47 $code = strtok($line, " \t"); | |
| 48 if ($code != 211) die("Error code $code from $peer[address]\n"); | |
| 49 strtok(" \t"); //number of articles | |
| 50 $low = strtok(" \t"); | |
| 51 $high = strtok(" \t"); | |
| 52 strtok(" \t"); //group name | |
| 53 if ($low != $peergroup['low'] || $high != $peergroup['high'] || $peergroup['low'] === NULL || $peergroup['high'] === NULL) { | |
| 54 for ($i = $low; $i <= $high; $i++) { | |
| 55 if ($i >= $peergroup['low'] && $i <= $peergroup['high'] && $peergroup['low'] !== NULL && $peergroup['high'] !== NULL) continue; | |
| 56 nntp_writeline($socket, 'STAT '.$i); | |
| 57 $line = nntp_readline($socket); | |
| 58 $code = strtok($line, " \t"); | |
| 59 if ($code == 423) { | |
| 60 print("Gap in article numbering at $i\n"); | |
| 61 continue; | |
| 62 } | |
| 63 if ($code != 223) die("Error code $code from $peer[address]\n"); | |
| 64 strtok(" \t"); //article number | |
| 65 $messageid = strtok(" \t"); | |
| 66 if ($messageid[0] != '<' || $messageid[strlen($messageid)-1] != '>') die("Malformed message ID $messageid\n"); | |
| 67 $messageid = substr($messageid, 1, -1); | |
| 68 $message = $db->evalRowAssoc('SELECT * FROM `messages` WHERE `messageid` = ?', $messageid); | |
| 69 if ($message) { | |
| 70 $groupmessage = $db->evalRowAssoc('SELECT * FROM `groupmessages` WHERE `group` = ? AND `message` = ?', array($group['id'], $message['id'])); | |
| 71 if (!$groupmessage) { | |
| 72 $db->insert('INSERT INTO `groupmessages` (`group`, `message`) VALUES (?, ?)', array($group['id'], $message['id'])); | |
| 73 } | |
| 74 } else { | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
75 nntp_writeline($socket, 'ARTICLE '.$i); |
| 0 | 76 $line = nntp_readline($socket); |
| 77 $code = strtok($line, " \t"); | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
78 if ($code != 220) die("Error code $code from $peer[address]\n"); |
| 0 | 79 strtok(" \t"); //article number |
| 80 $lines = nntp_readlines($socket); | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
81 try { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
82 nntp_article_store($lines); |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
83 } catch (Exception $ex) { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
84 writelog($ex->getMessage()); |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
85 } |
| 0 | 86 } |
| 87 } | |
| 88 $db->update('UPDATE `peergroups` SET `low` = ?, `high` = ? WHERE `peer` = ? AND `group` = ?', array($low, $high, $peergroup['peer'], $peergroup['group'])); | |
| 89 } | |
| 90 } | |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
91 while ($peer['post']) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
92 if ($peer['lastposted'] === NULL) { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
93 $articles = $db->evalAllAssoc('SELECT * FROM `messages` LIMIT 10'); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
94 } else { |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
95 $articles = $db->evalAllAssoc('SELECT * FROM `messages` WHERE `id` > ? LIMIT 10', $peer['lastposted']); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
96 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
97 if (!count($articles)) break; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
98 foreach ($articles as $article) { |
|
9
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
99 $dopost = FALSE; |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
100 if ($peer['post'] == 2) { |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
101 nntp_writeline($socket, 'IHAVE <'.$article['messageid'].'>'); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
102 $line = nntp_readline($socket); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
103 $code = strtok($line, " \t"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
104 if ($code == 335) $dopost = TRUE; |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
105 elseif ($code != 435) $peer['post'] = 1; |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
106 } |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
107 if ($peer['post'] != 2) { |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
108 nntp_writeline($socket, 'STAT <'.$article['messageid'].'>'); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
109 $line = nntp_readline($socket); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
110 $code = strtok($line, " \t"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
111 if ($code == 430) $dopost = TRUE; |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
112 elseif ($code != 223) die("Error code $code from $peer[address]\n"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
113 if ($dopost) { |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
114 nntp_writeline($socket, 'POST'); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
115 $line = nntp_readline($socket); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
116 $code = strtok($line, " \t"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
117 if ($code != 340) die("Error code $code from $peer[address]\n"); |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
118 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
119 } |
|
9
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
120 if ($dopost) { |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
121 foreach (explode("\r\n", $article['header']) as $line) nntp_writeline_data($socket, $line); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
122 nntp_writeline($socket, ''); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
123 foreach (explode("\r\n", $article['body']) as $line) nntp_writeline_data($socket, $line); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
124 nntp_writeline($socket, '.'); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
125 $line = nntp_readline($socket); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
126 $code = strtok($line, " \t"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
127 if ($code != 240 && $code != 235) print("Article $article[messageid] was not accepted ($code)\n"); |
|
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
128 } |
|
2
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
129 if ($article['id'] > $peer['lastposted']) $peer['lastposted'] = $article['id']; |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
130 } |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
131 $db->update('UPDATE `peers` SET `lastposted` = ? WHERE `id` = ?', array($peer['lastposted'], $peer['id'])); |
|
40e545510a57
Added support for POSTing to the server, added readme and todo, added support for upstream synchronization using POST
Ivo Smits <Ivo@UCIS.nl>
parents:
1
diff
changeset
|
132 } |
| 0 | 133 nntp_writeline($socket, 'QUIT'); |
|
9
ae0c67d72670
Bugfix in server POST code, add support for client side IHAVE and fall back to STAT-before-POST
Ivo Smits <Ivo@UCIS.nl>
parents:
8
diff
changeset
|
134 nntp_readline($socket); |
| 0 | 135 fclose($socket); |
| 136 } | |
| 137 | |
|
3
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
138 function writelog($line) { |
|
0dcdb73cbcbf
Increased article body length to 16MB in database scheme, cleaned up the code, added a script to forcefully reload an individual article
Ivo Smits <Ivo@UCIS.nl>
parents:
2
diff
changeset
|
139 print($line."\n"); |
|
1
61fac319ca3e
Update Path and Xref header fields while fetching messages, added database schema export
Ivo Smits <Ivo@UCIS.nl>
parents:
0
diff
changeset
|
140 } |