From 4e85b423354d8b715d12540fbd2ac79a66de3399 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 19:56:22 +0100 Subject: add download-command cli interface & config --- command/download.php | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 command/download.php (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php new file mode 100644 index 0000000..8cc2f53 --- /dev/null +++ b/command/download.php @@ -0,0 +1,28 @@ +getSlug()); + + if(isset($conf['MAX_CONFERENCE_AGE'])) + { + date_diff() + return time() >= $this->endsAt(); + } + +} -- cgit v1.2.3 From 8beec6fda376b842a9ddb7a2425829690f0a8b48 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 20:27:35 +0100 Subject: use DateTime objects for ends/startsAt --- command/download.php | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index 8cc2f53..cb0987d 100644 --- a/command/download.php +++ b/command/download.php @@ -15,14 +15,32 @@ if(isset($conf['REQUIRE_USER'])) } } -foreach (Conferences::getConferences() as $conference) +$conferences = Conferences::getConferences(); + +if(isset($conf['MAX_CONFERENCE_AGE'])) { - stdout('== %s ==', $conference->getSlug()); + $months = intval($conf['MAX_CONFERENCE_AGE']); + $conferencesAfter = new DateTime(); + $conferencesAfter->sub(new DateInterval('P'.$months.'D')); - if(isset($conf['MAX_CONFERENCE_AGE'])) - { - date_diff() - return time() >= $this->endsAt(); - } + stdout('Filtering before %s', $conferencesAfter->format('Y-m-d')); + $conferences = array_filter($conferences, function($conference) use ($conferencesAfter) { + $isBefore = $conference->endsAt() < $conferencesAfter; + + if($isBefore) { + stdout( + ' %s: %s', + $conference->endsAt()->format('Y-m-d'), + $conference->getSlug() + ); + } + return !$isBefore; + }); +} + +stdout(''); +foreach ($conferences as $conference) +{ + stdout('== %s ==', $conference->getSlug()); } -- cgit v1.2.3 From 6e0f7423769174b83d6b5618db7c33d54967099a Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 21:17:30 +0100 Subject: download files of forced-open conferences --- command/download.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index cb0987d..d0999b4 100644 --- a/command/download.php +++ b/command/download.php @@ -23,8 +23,19 @@ if(isset($conf['MAX_CONFERENCE_AGE'])) $conferencesAfter = new DateTime(); $conferencesAfter->sub(new DateInterval('P'.$months.'D')); - stdout('Filtering before %s', $conferencesAfter->format('Y-m-d')); + stdout('Skipping Conferences before %s', $conferencesAfter->format('Y-m-d')); $conferences = array_filter($conferences, function($conference) use ($conferencesAfter) { + if($conference->isOpen()) + { + stdout( + ' %s: %s', + '---open---', + $conference->getSlug() + ); + + return true; + } + $isBefore = $conference->endsAt() < $conferencesAfter; if($isBefore) { -- cgit v1.2.3 From 284878f0f9917b55359cd464b86508cacc472653 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 21:46:01 +0100 Subject: add cache-file getters and download preparations --- command/download.php | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index d0999b4..8deb454 100644 --- a/command/download.php +++ b/command/download.php @@ -54,4 +54,72 @@ stdout(''); foreach ($conferences as $conference) { stdout('== %s ==', $conference->getSlug()); + + $relive = $conference->getRelive(); + if($relive->isEnabled()) + { + download( + 'relive-json', + $conference, + $relive->getJsonUrl(), + $relive->getJsonCache() + ); + } + + $schedule = $conference->getSchedule(); + if($schedule->isEnabled()) + { + download( + 'schedule-xml', + $conference, + $schedule->getScheduleUrl(), + $schedule->getScheduleCache() + ); + } + + foreach($conference->getExtraFiles() as $file) + { + download( + 'extra-file', + $conference, + $file, + get_file_cache($conference, $file) + ); + } +} + + + + +function get_file_cache($conference, $url) +{ + $info = parse_url($url); + $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_'); + $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_'); + return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url)); +} + +function download($what, $conference, $url, $cache) +{ + stdout( + ' downloading %s from %s to %s', + $what, + $url, + $cache + ); + if(!do_download($url, $cache)) + { + stderr( + '!! download %s for conference %s from %s to %s failed miserably !!', + $what, + $conference->getSlug(), + $url, + $cache + ); + } +} + +function do_download($url, $cache) +{ + return true; } -- cgit v1.2.3 From 3384e963da66e9089b71c546750c8981be8c6ed8 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:11:47 +0100 Subject: change storage paths according to issue description --- command/download.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index 8deb454..17ae7c9 100644 --- a/command/download.php +++ b/command/download.php @@ -77,13 +77,13 @@ foreach ($conferences as $conference) ); } - foreach($conference->getExtraFiles() as $file) + foreach($conference->getExtraFiles() as $filename => $url) { download( 'extra-file', $conference, - $file, - get_file_cache($conference, $file) + $url, + get_file_cache($conference, $filename) ); } } @@ -91,12 +91,9 @@ foreach ($conferences as $conference) -function get_file_cache($conference, $url) +function get_file_cache($conference, $filename) { - $info = parse_url($url); - $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_'); - $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_'); - return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url)); + return joinpath([$GLOBALS['BASEDIR'], 'configs/conferences', $conference->getSlug(), $filename]); } function download($what, $conference, $url, $cache) -- cgit v1.2.3 From f4b2cb58b2e62e365103cbcffc3d8c2999b3f300 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:12:36 +0100 Subject: warn on old-style paths --- command/download.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index 17ae7c9..c9fc5cd 100644 --- a/command/download.php +++ b/command/download.php @@ -98,6 +98,18 @@ function get_file_cache($conference, $filename) function download($what, $conference, $url, $cache) { + $info = parse_url($url); + if(!isset($info['scheme']) || !isset($info['host'])) + { + stderr( + ' !! %s url for conference %s does look like an old-style path: "%s". please update to a full http/https url', + $what, + $conference->getSlug(), + $url + ); + return false; + } + stdout( ' downloading %s from %s to %s', $what, -- cgit v1.2.3 From e7b0c27b54cdbcf248db191bcd34d5b8b4de2e99 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:12:56 +0100 Subject: improve log formatting --- command/download.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index c9fc5cd..4018a9a 100644 --- a/command/download.php +++ b/command/download.php @@ -119,13 +119,14 @@ function download($what, $conference, $url, $cache) if(!do_download($url, $cache)) { stderr( - '!! download %s for conference %s from %s to %s failed miserably !!', + ' !! download %s for conference %s from %s to %s failed miserably !!', $what, $conference->getSlug(), $url, $cache ); } + return true; } function do_download($url, $cache) -- cgit v1.2.3 From 63967f91dacca321a63d91be4dd76d0ed3eb4956 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:13:14 +0100 Subject: implement actual downloading --- command/download.php | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index 4018a9a..486c7af 100644 --- a/command/download.php +++ b/command/download.php @@ -131,5 +131,33 @@ function download($what, $conference, $url, $cache) function do_download($url, $cache) { + $handle = curl_init($url); + curl_setopt_array($handle, [ + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_MAXREDIRS => 10, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_SSL_VERIFYPEER => false, /* accept all certificates, even self-signed */ + CURLOPT_SSL_VERIFYHOST => 2, /* verify hostname is in cert */ + CURLOPT_CONNECTTIMEOUT => 3, /* connect-timeout in seconds */ + CURLOPT_TIMEOUT => 5, /* transfer timeout im seconds */ + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, + CURLOPT_REFERER => 'https://streaming.media.ccc.de/', + CURLOPT_USERAGENT => '@c3voc Streaming-Website Downloader-Cronjob, Contact voc AT c3voc DOT de in case of problems. Might the Winkekatze be with you', + ]); + + $return = curl_exec($handle); + $info = curl_getinfo($handle); + curl_close($handle); + + if($info['http_code'] != 200) + return false; + + $tempfile = tempnam(dirname($cache), 'dl-'); + if(false === file_put_contents($tempfile, $return)) + return false; + + chmod($tempfile, 0644); + rename($tempfile, $cache); + return true; } -- cgit v1.2.3 From a77d28fb7252c35b59aa4993cd25c4d9eaeb3de3 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 18 Dec 2016 10:51:02 +0100 Subject: download eventcalender upcoming, too --- command/download.php | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) (limited to 'command/download.php') diff --git a/command/download.php b/command/download.php index 486c7af..9ec042e 100644 --- a/command/download.php +++ b/command/download.php @@ -58,7 +58,7 @@ foreach ($conferences as $conference) $relive = $conference->getRelive(); if($relive->isEnabled()) { - download( + download_for_conference( 'relive-json', $conference, $relive->getJsonUrl(), @@ -69,7 +69,7 @@ foreach ($conferences as $conference) $schedule = $conference->getSchedule(); if($schedule->isEnabled()) { - download( + download_for_conference( 'schedule-xml', $conference, $schedule->getScheduleUrl(), @@ -79,7 +79,7 @@ foreach ($conferences as $conference) foreach($conference->getExtraFiles() as $filename => $url) { - download( + download_for_conference( 'extra-file', $conference, $url, @@ -88,6 +88,15 @@ foreach ($conferences as $conference) } } +stdout(''); +stdout('== eventkalender =='); +download( + 'eventkalender', + 'https://c3voc.de/eventkalender/events.json?filter=upcoming&streaming=yes', + joinpath([$GLOBALS['BASEDIR'], 'configs/upcoming.json']) +); + + @@ -96,7 +105,7 @@ function get_file_cache($conference, $filename) return joinpath([$GLOBALS['BASEDIR'], 'configs/conferences', $conference->getSlug(), $filename]); } -function download($what, $conference, $url, $cache) +function download_for_conference($what, $conference, $url, $cache) { $info = parse_url($url); if(!isset($info['scheme']) || !isset($info['host'])) @@ -129,6 +138,28 @@ function download($what, $conference, $url, $cache) return true; } +function download($what, $url, $cache) +{ + stdout( + ' downloading %s from %s to %s', + $what, + $url, + $cache + ); + $resp = do_download($url, $cache); + if($resp !== true) + { + stderr( + ' !! download %s from %s to %s failed miserably: %s !!', + $what, + $url, + $cache, + $resp + ); + } + return true; +} + function do_download($url, $cache) { $handle = curl_init($url); @@ -141,7 +172,6 @@ function do_download($url, $cache) CURLOPT_CONNECTTIMEOUT => 3, /* connect-timeout in seconds */ CURLOPT_TIMEOUT => 5, /* transfer timeout im seconds */ CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, - CURLOPT_REFERER => 'https://streaming.media.ccc.de/', CURLOPT_USERAGENT => '@c3voc Streaming-Website Downloader-Cronjob, Contact voc AT c3voc DOT de in case of problems. Might the Winkekatze be with you', ]); @@ -150,11 +180,14 @@ function do_download($url, $cache) curl_close($handle); if($info['http_code'] != 200) - return false; + return 'http-code = '.$info['http_code']; $tempfile = tempnam(dirname($cache), 'dl-'); + if(!$tempfile) + return 'could not create tempfile in '.dirname($cache); + if(false === file_put_contents($tempfile, $return)) - return false; + return 'could write data into tempfile '.$tempfile; chmod($tempfile, 0644); rename($tempfile, $cache); -- cgit v1.2.3