1 ////////////////////////////////////////////////////////////////////////
2 // FILE: httpup.cpp
3 // AUTHOR: Johannes Winkelmann, jw@tks6.net
4 // COPYRIGHT: (c) 2002-2005 by Johannes Winkelmann
5 // ---------------------------------------------------------------------
6 // This program is free software; you can redistribute it and/or modify
7 // it under the terms of the GNU General Public License as published by
8 // the Free Software Foundation; either version 2 of the License, or
9 // (at your option) any later version.
10 ////////////////////////////////////////////////////////////////////////
11
12 #include <iostream>
13
14 #include <sys/types.h>
15 #include <sys/stat.h>
16 #include <unistd.h>
17 #include <libgen.h>
18 #include <dirent.h>
19 #include <cstring>
20 #include <cstdlib>
21
22 #include "fileutils.h"
23 #include "httpup.h"
24 #include "configparser.h"
25
26 using namespace std;
27
28 const string HttpUp::DEFAULT_REPOFILE = "REPO";
29 const string HttpUp::REPOCURRENTFILEOLD = "REPO.CURRENT";
30 const string HttpUp::REPOCURRENTFILE = ".httpup-repo.current";
31 const string HttpUp::URLINFO = ".httpup-urlinfo";
32
33 const int HttpUp::DEFAULT_TIMEOUT = 60;
34
35 HttpUp::HttpUp(const HttpupArgparser& argParser,
36 const string& url, const string& target,
37 const string& fragment, const string& repoFile,
38 bool verifyMd5)
39 : m_baseDirectory(target),
40 m_remoteUrl(url),
41 m_fragment(fragment),
42 m_argParser(argParser),
43 m_verifyMd5(verifyMd5)
44 {
45 if (repoFile != "") {
46 m_repoFile = repoFile;
47 } else {
48 m_repoFile = DEFAULT_REPOFILE;
49 }
50 }
51
52
53 int HttpUp::parseCurrent()
54 {
55 FILE* fp = fopen((m_baseDirectory+"/"+REPOCURRENTFILE).c_str(), "r");
56 if (!fp) {
57 // TODO: remove in 0.3.1
58 fp = fopen((m_baseDirectory+"/"+REPOCURRENTFILEOLD).c_str(), "r");
59 if (!fp) {
60 return -1;
61 }
62 }
63 char input[512];
64
65 while (fgets(input, 512, fp)) {
66 input[strlen(input)-1] = '\0';
67 m_actions[string(input)] = REMOVE;
68 }
69
70 return 0;
71 }
72
73 int HttpUp::findDiff()
74 {
75 FILE* fp = fopen((m_baseDirectory + m_repoFile).c_str(), "r");
76 if (!fp) {
77 cerr << "Couldn't open " << m_repoFile << endl;
78 return -1;
79 }
80 char input[512];
81 struct stat info;
82
83 string fileToStat;
84 while (fgets(input, 512, fp)) {
85 input[strlen(input)-1] = '\0';
86 if (input[0] == 'd') {
87
88 string dir = input+2;
89
90 if (m_fragment != "" &&
91 dir.substr(0, m_fragment.length()) != m_fragment) {
92 // doesn't start with fragment
93 continue;
94 }
95
96 if (m_fragment == dir) {
97 continue;
98 }
99
100 if (m_fragment != "") {
101 if (dir.substr(0, m_fragment.length()) == m_fragment &&
102 dir.length() > m_fragment.length()+1 &&
103 dir[m_fragment.length()] == '/') {
104 // strip; matching but hierarchy
105
106 dir = dir.substr(m_fragment.length()+1);
107 if (dir.length() == 0) {
108 continue;
109 }
110 } else {
111 // strip: fragment is only a substring of dir
112 continue;
113 }
114 }
115
116 m_remoteFiles.push_back(dir);
117 fileToStat = m_baseDirectory + (dir);
118 if (stat(fileToStat.c_str(), &info) == 0) {
119 // dir exists
120 if (!S_ISDIR(info.st_mode)) {
121 m_actions[dir] = REPLACE_FILE_WITH_DIR;
122 } else {
123 m_actions[dir] = NOP;
124 }
125 } else {
126 m_actions[dir] = DIR_CREATE;
127 }
128 } else {
129 int fileNameOffset = 2 + 32 + 1;
130 // 0+2+32+1 means
131 // +2 skip the "f:" string
132 // +32 skip the md5 string
133 // +1 skip the separator (':') between fileName and md5
134
135 string file = input+fileNameOffset;
136 if (m_fragment != "" &&
137 file.substr(0, m_fragment.length()) != m_fragment) {
138 // doesn't start with fragment
139 continue;
140 }
141
142 if (m_fragment != "") {
143
144 if (file.substr(0, m_fragment.length()) == m_fragment &&
145 file.length() > m_fragment.length()+1 &&
146 file[m_fragment.length()] == '/') {
147
148 file = file.substr(m_fragment.length()+1);
149 } else {
150 // skip; fragment is only a substring
151 continue;
152 }
153 }
154
155
156 m_remoteFiles.push_back(file);
157 fileToStat = m_baseDirectory + (file);
158 if (stat(fileToStat.c_str(), &info) == 0) {
159
160 if (S_ISDIR(info.st_mode)) {
161 m_actions[file] = REPLACE_DIR_WITH_FILE;
162 } else {
163 // file exists
164 unsigned char result[16];
165 bool diff = false;
166 if (FileUtils::fmd5sum(fileToStat, result)) {
167 input[2+32] = '\0';
168 diff = verifyMd5sum(input+2, result);
169 }
170 if (diff) {
171 m_actions[file] = FILE_GET;
172 } else {
173 m_actions[file] = NOP;
174 }
175 }
176 } else {
177 m_actions[file] = NEW_FILE_GET;
178 }
179
180 if (m_verifyMd5) {
181 m_md5sums[file] = string(input+2);
182 }
183 }
184 }
185 fclose(fp);
186
187 return 0;
188 }
189
190 bool HttpUp::verifyMd5sum(const char* input, unsigned char result[16])
191 {
192 static char hexNumbers[] = {'0','1','2','3','4','5','6','7',
193 '8','9','a','b','c','d','e','f'};
194 bool diff = false;
195
196 unsigned char high, low;
197 for (int i = 0; i < 16; ++i) {
198 high = (result[i] & 0xF0) >> 4;
199 low = result[i] & 0xF;
200 if (*(input+2*i) - hexNumbers[high] ||
201 *(input+2*i+1) - hexNumbers[low]) {
202 diff = true;
203 break;
204 }
205 }
206
207 return diff;
208 }
209
210 int HttpUp::exec(ExecType type)
211 {
212 struct stat info;
213 if (stat(m_baseDirectory.c_str(), &info)) {
214 if (FileUtils::mktree(m_baseDirectory.c_str())) {
215 cerr << "Failed to create base directory "
216 << m_baseDirectory << endl;
217 return -1;
218 }
219 }
220
221 Config config;
222 ConfigParser::parseConfig("/etc/httpup.conf", config);
223
224
225 // TODO: check return values.
226 CURL *curl;
227 curl_global_init(CURL_GLOBAL_ALL);
228 curl = curl_easy_init();
229
230 struct curl_slist *headers=NULL;
231 headers = curl_slist_append(headers, "Cache-Control: no-cache, must-revalidate");
232 headers = curl_slist_append(headers, "Pragma: no-cache");
233
234
235 char errorBuffer[CURL_ERROR_SIZE];
236 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
237 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorBuffer);
238 curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
239 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
240 if (m_argParser.isSet(HttpupArgparser::OPT_INSECURE_SSL)) {
241 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
242 }
243
244
245 long timeout = DEFAULT_TIMEOUT;
246 if (config.operationTimeout != "") {
247 char* end = 0;
248 long config_timeout = 0;
249 config_timeout = strtol(config.operationTimeout.c_str(), &end, 10);
250 if (*end == 0) {
251 timeout = config_timeout;
252 }
253 }
254 curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
255
256
257 // proxy, proxy auth
258 if (config.proxyHost != "") {
259 curl_easy_setopt(curl, CURLOPT_PROXY, config.proxyHost.c_str());
260 }
261
262 if (config.proxyPort != "") {
263 long port = atol(config.proxyPort.c_str());
264 curl_easy_setopt(curl, CURLOPT_PROXYPORT, port);
265 }
266
267 string usrpwd;
268 if (config.proxyUser != "" || config.proxyPassword != "") {
269 usrpwd = config.proxyUser + ":" + config.proxyPassword;
270 curl_easy_setopt(curl, CURLOPT_PROXYUSERPWD, usrpwd.c_str());
271 }
272
273
274 #if 0
275 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
276 #endif
277
278 if (!curl) {
279 cerr << "Failed to initialize CURL engine" << endl;
280 return -1;
281 }
282
283 cout << "Connecting to " << m_remoteUrl << endl;
284 int ret = syncOrReturn(curl, errorBuffer);
285
286 curl_easy_cleanup(curl);
287
288 if (ret == 0) {
289
290 if (type == TYPE_SYNC) {
291 saveRepoCurrent();
292 } else if (type == TYPE_COPY){
293 unlink((m_baseDirectory+m_repoFile).c_str());
294 }
295
296
297 if (type == TYPE_SYNC) {
298 FILE* fp = fopen((m_baseDirectory+"/"+URLINFO).c_str(), "w");
299 if (fp) {
300 fprintf(fp, "%s#%s",
301 m_remoteUrl.c_str(), m_fragment.c_str());
302 fclose(fp);
303 } else {
304 cerr << "Failed to store urlinfo" << endl;
305 }
306 }
307 }
308
309 return ret;
310 }
311
312 int HttpUp::syncOrReturn(CURL* curl, char* curlErrorBuffer)
313 {
314
315 if (getRemoteRepoFile(curl, curlErrorBuffer) != 0) {
316 cerr << "Failed to get remote repo file" << endl;
317 return -1;
318 }
319
320 string collectionName =
321 m_baseDirectory.substr(0, m_baseDirectory.length()-1);
322 string::size_type pos = collectionName.rfind("/");
323 if (pos != string::npos) {
324 collectionName = collectionName.substr(pos+1);
325 }
326
327 cout << "Updating collection " << collectionName << endl;
328
329 // compare with local directory
330 if (parseCurrent() != 0) {
331 // -- also "fails" the first time...
332 // cerr << "Failed to parse local directory" << endl;
333 // return -1;
334 }
335
336 if (findDiff() != 0) {
337 cerr << "Failed to check for differences" << endl;
338 return -1;
339 }
340
341 #if 0
342 if (m_actions.size() == 0) {
343 cerr << "No matches found for fragment " << m_fragment << endl;
344 return -1;
345 }
346 #endif
347
348
349 return getChangedFiles(collectionName, curl, curlErrorBuffer);;
350 }
351
352 void HttpUp::saveRepoCurrent()
353 {
354 // save current
355 FILE* current = fopen((m_baseDirectory + REPOCURRENTFILE).c_str(), "w");
356 if (!current) {
357 cerr << "Couldn't open "
358 << m_baseDirectory << REPOCURRENTFILE << " for writing" << endl;
359 } else {
360 list<string>::iterator cit = m_remoteFiles.begin();
361 for (; cit != m_remoteFiles.end(); ++cit) {
362 fprintf(current, "%s\n", cit->c_str());
363 }
364 fclose(current);
365 }
366
367 // TODO: remove in 0.3.1
368 FILE* fp = fopen((m_baseDirectory+REPOCURRENTFILEOLD).c_str(), "r");
369 if (fp) {
370 fclose(fp);
371 unlink((m_baseDirectory+REPOCURRENTFILEOLD).c_str());
372 }
373
374 unlink((m_baseDirectory+m_repoFile).c_str());
375 cout << "Finished successfully" << endl;
376 }
377
378 int HttpUp::getChangedFiles(const string& collectionName, CURL* curl,
379 char* curlErrorBuffer)
380 {
381 int errors = 0;
382
383 string fragment = m_fragment;
384 if (fragment != "") {
385 fragment += "/";
386 }
387
388 // synchronize
389 map<string, Action>::iterator it = m_actions.begin();
390 for (; it != m_actions.end(); ++it) {
391
392 if (it->first.substr(0, 3) == "../" ||
393 it->first.find("/../") != string::npos) {
394 cerr << " WARNING: Malicious path in remote REPO file: "
395 << it->first << endl;
396 continue;
397 }
398
399 if (it->second == DIR_CREATE) {
400 cout << " Checkout: "
401 << collectionName << "/" << it->first << endl;
402
403 mkdir((m_baseDirectory+it->first).c_str(), 0755);
404 } else if (it->second == NEW_FILE_GET || it->second == FILE_GET) {
405 if (it->second == NEW_FILE_GET) {
406 cout << " Checkout: "
407 << collectionName << "/" << it->first << endl;
408 } else if (it->second == FILE_GET) {
409 cout << " Edit: "
410 << collectionName << "/" << it->first << endl;
411 }
412
413 string fileName = it->first;
414 if (m_argParser.isSet(HttpupArgparser::OPT_ENCODE)) {
415 char* p = curl_escape(fileName.c_str(), fileName.length());
416 fileName = p;
417 curl_free(p);
418 }
419
420 string fileURL = m_remoteUrl+fragment+fileName;
421 curl_easy_setopt(curl, CURLOPT_URL, fileURL.c_str());
422
423 FILE* dlFile = fopen((m_baseDirectory+it->first).c_str(), "w");
424 if (!dlFile) {
425 cout << " Failed to open " << it->first
426 << " for writing" <<endl;
427 } else {
428 curl_easy_setopt(curl, CURLOPT_FILE, dlFile);
429 CURLcode res = curl_easy_perform(curl);
430 if (res) {
431 cout << "Failed to download " << fileURL
432 << ": " << curlErrorBuffer << endl;
433 }
434 fclose(dlFile);
435 }
436
437 if (m_verifyMd5) {
438 unsigned char result[16];
439 if (FileUtils::fmd5sum(m_baseDirectory+it->first, result)) {
440 bool diff =
441 verifyMd5sum(m_md5sums[it->first.c_str()].c_str(),
442 result);
443
444 if (diff) {
445 cerr << "Bad md5sum after download for "
446 << it->first << endl;
447 ++errors;
448 }
449 } else {
450 ++errors;
451 }
452 }
453
454 } else if (it->second == REPLACE_DIR_WITH_FILE) {
455 cout << " Cowardly refusing to overwrite directory '"
456 << m_baseDirectory+it->first
457 << "' with a file" << endl;
458 continue;
459 } else if (it->second == REPLACE_FILE_WITH_DIR) {
460 cout << " Remove: "
461 << collectionName << "/" << it->first
462 << " (file)"
463 << endl;
464 int ret = FileUtils::deltree((m_baseDirectory+it->first).c_str());
465
466 if (ret == 0) {
467 cout << " Checkout: "
468 << collectionName << "/" << it->first << endl;
469 mkdir((m_baseDirectory+it->first).c_str(), 0755);
470 }
471 } else if (it->second == REMOVE) {
472 cout << " Delete: "
473 << collectionName << "/" << it->first << endl;
474
475 if (FileUtils::deltree((m_baseDirectory+it->first).c_str())) {
476 cout << " Failed to remove " << it->first << endl;
477 m_remoteFiles.push_back(it->first);
478 }
479 }
480 }
481
482 return errors;
483 }
484
485 int HttpUp::getRemoteRepoFile(CURL* curl, char* curlErrorBuffer)
486 {
487 // download repo
488 FILE* dlFile = 0;
489 string fileName = m_repoFile;
490 if (m_argParser.isSet(HttpupArgparser::OPT_ENCODE)) {
491 char* p = curl_escape(fileName.c_str(), fileName.length());
492 fileName = p;
493 curl_free(p);
494 }
495 string repoURL = m_remoteUrl + fileName;
496
497 curl_easy_setopt(curl, CURLOPT_URL, repoURL.c_str());
498 dlFile = fopen((m_baseDirectory+m_repoFile).c_str(), "w");
499 if (!dlFile) {
500 cout << " Failed to open " << m_repoFile << " for writing" << endl;
501 } else {
502
503 curl_easy_setopt(curl, CURLOPT_FILE, dlFile);
504 CURLcode res = curl_easy_perform(curl);
505 if (res) {
506 cerr << " Failed to download " << m_repoFile
507 << ": " << curlErrorBuffer << endl;
508 return -1;
509 }
510 fclose(dlFile);
511 }
512
513
514 return 0;
515 }
|
