From 4e85b423354d8b715d12540fbd2ac79a66de3399 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 19:56:22 +0100 Subject: add download-command cli interface & config --- command/download.php | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 command/download.php (limited to 'command') diff --git a/command/download.php b/command/download.php new file mode 100644 index 0000000..8cc2f53 --- /dev/null +++ b/command/download.php @@ -0,0 +1,28 @@ +getSlug()); + + if(isset($conf['MAX_CONFERENCE_AGE'])) + { + date_diff() + return time() >= $this->endsAt(); + } + +} -- cgit v1.3.1 From 8beec6fda376b842a9ddb7a2425829690f0a8b48 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 20:27:35 +0100 Subject: use DateTime objects for ends/startsAt --- command/download.php | 32 +++++++++++++++++++++++++------- config.php | 2 +- model/Conference.php | 16 ++++++++++++---- model/Conferences.php | 2 +- 4 files changed, 39 insertions(+), 13 deletions(-) (limited to 'command') diff --git a/command/download.php b/command/download.php index 8cc2f53..cb0987d 100644 --- a/command/download.php +++ b/command/download.php @@ -15,14 +15,32 @@ if(isset($conf['REQUIRE_USER'])) } } -foreach (Conferences::getConferences() as $conference) +$conferences = Conferences::getConferences(); + +if(isset($conf['MAX_CONFERENCE_AGE'])) { - stdout('== %s ==', $conference->getSlug()); + $months = intval($conf['MAX_CONFERENCE_AGE']); + $conferencesAfter = new DateTime(); + $conferencesAfter->sub(new DateInterval('P'.$months.'D')); - if(isset($conf['MAX_CONFERENCE_AGE'])) - { - date_diff() - return time() >= $this->endsAt(); - } + stdout('Filtering before %s', $conferencesAfter->format('Y-m-d')); + $conferences = array_filter($conferences, function($conference) use ($conferencesAfter) { + $isBefore = $conference->endsAt() < $conferencesAfter; + + if($isBefore) { + stdout( + ' %s: %s', + $conference->endsAt()->format('Y-m-d'), + $conference->getSlug() + ); + } + return !$isBefore; + }); +} + +stdout(''); +foreach ($conferences as $conference) +{ + stdout('== %s ==', $conference->getSlug()); } diff --git a/config.php b/config.php index 57be966..1b5e84f 100644 --- a/config.php +++ b/config.php @@ -61,5 +61,5 @@ $GLOBALS['CONFIG']['DOWNLOAD'] = [ * * Auskommentieren, um alle Konferenzen zu beachten */ - 'MAX_CONFERENCE_AGE' => 30 /* Tage */, + 'MAX_CONFERENCE_AGE' => 14 /* Tage */, ]; diff --git a/model/Conference.php b/model/Conference.php index 07d72dd..ae5ab41 100644 --- a/model/Conference.php +++ b/model/Conference.php @@ -33,11 +33,17 @@ class Conference extends ModelBase } public function startsAt() { - return $this->get('CONFERENCE.STARTS_AT'); + if(!$this->has('CONFERENCE.STARTS_AT')) + return null; + + return DateTime::createFromFormat('U', $this->get('CONFERENCE.STARTS_AT')); } public function endsAt() { - return $this->get('CONFERENCE.ENDS_AT'); + if(!$this->has('CONFERENCE.ENDS_AT')) + return null; + + return DateTime::createFromFormat('U', $this->get('CONFERENCE.ENDS_AT')); } public function hasBegun() { @@ -62,7 +68,8 @@ class Conference extends ModelBase } if($this->has('CONFERENCE.STARTS_AT')) { - return time() >= $this->get('CONFERENCE.STARTS_AT'); + $now = new DateTime('now'); + return $now >= $this->startsAt(); } else { return true; } @@ -84,7 +91,8 @@ class Conference extends ModelBase } if($this->has('CONFERENCE.ENDS_AT')) { - return time() >= $this->get('CONFERENCE.ENDS_AT'); + $now = new DateTime('now'); + return $now >= $this->endsAt(); } else { return false; } diff --git a/model/Conferences.php b/model/Conferences.php index 2dc9f35..8de4029 100644 --- a/model/Conferences.php +++ b/model/Conferences.php @@ -48,7 +48,7 @@ class Conferences $sorted = Conferences::getConferences(); usort($sorted, function($a, $b) { - return $b->startsAt() - $a->endsAt(); + return $b->startsAt() > $a->endsAt() ? 1 : -1; }); return $sorted; -- cgit v1.3.1 From 6e0f7423769174b83d6b5618db7c33d54967099a Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 21:17:30 +0100 Subject: download files of forced-open conferences --- command/download.php | 13 ++++++++++++- model/Conference.php | 6 +++++- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'command') diff --git a/command/download.php b/command/download.php index cb0987d..d0999b4 100644 --- a/command/download.php +++ b/command/download.php @@ -23,8 +23,19 @@ if(isset($conf['MAX_CONFERENCE_AGE'])) $conferencesAfter = new DateTime(); $conferencesAfter->sub(new DateInterval('P'.$months.'D')); - stdout('Filtering before %s', $conferencesAfter->format('Y-m-d')); + stdout('Skipping Conferences before %s', $conferencesAfter->format('Y-m-d')); $conferences = array_filter($conferences, function($conference) use ($conferencesAfter) { + if($conference->isOpen()) + { + stdout( + ' %s: %s', + '---open---', + $conference->getSlug() + ); + + return true; + } + $isBefore = $conference->endsAt() < $conferencesAfter; if($isBefore) { diff --git a/model/Conference.php b/model/Conference.php index ae5ab41..be62fdb 100644 --- a/model/Conference.php +++ b/model/Conference.php @@ -19,7 +19,7 @@ class Conference extends ModelBase } public function isPreviewEnabled() { - if($GLOBALS['forceopen']) + if(@$GLOBALS['forceopen']) return true; if($this->has('PREVIEW_DOMAIN') && ($this->get('PREVIEW_DOMAIN') == $_SERVER['SERVER_NAME'])) @@ -32,6 +32,10 @@ class Conference extends ModelBase return !$this->hasBegun() || $this->hasEnded(); } + public function isOpen() { + return !$this->isClosed(); + } + public function startsAt() { if(!$this->has('CONFERENCE.STARTS_AT')) return null; -- cgit v1.3.1 From 284878f0f9917b55359cd464b86508cacc472653 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sun, 11 Dec 2016 21:46:01 +0100 Subject: add cache-file getters and download preparations --- command/download.php | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ model/Conference.php | 4 ++++ model/Relive.php | 5 ++++ model/Schedule.php | 7 +++++- 4 files changed, 83 insertions(+), 1 deletion(-) (limited to 'command') diff --git a/command/download.php b/command/download.php index d0999b4..8deb454 100644 --- a/command/download.php +++ b/command/download.php @@ -54,4 +54,72 @@ stdout(''); foreach ($conferences as $conference) { stdout('== %s ==', $conference->getSlug()); + + $relive = $conference->getRelive(); + if($relive->isEnabled()) + { + download( + 'relive-json', + $conference, + $relive->getJsonUrl(), + $relive->getJsonCache() + ); + } + + $schedule = $conference->getSchedule(); + if($schedule->isEnabled()) + { + download( + 'schedule-xml', + $conference, + $schedule->getScheduleUrl(), + $schedule->getScheduleCache() + ); + } + + foreach($conference->getExtraFiles() as $file) + { + download( + 'extra-file', + $conference, + $file, + get_file_cache($conference, $file) + ); + } +} + + + + +function get_file_cache($conference, $url) +{ + $info = parse_url($url); + $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_'); + $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_'); + return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url)); +} + +function download($what, $conference, $url, $cache) +{ + stdout( + ' downloading %s from %s to %s', + $what, + $url, + $cache + ); + if(!do_download($url, $cache)) + { + stderr( + '!! download %s for conference %s from %s to %s failed miserably !!', + $what, + $conference->getSlug(), + $url, + $cache + ); + } +} + +function do_download($url, $cache) +{ + return true; } diff --git a/model/Conference.php b/model/Conference.php index be62fdb..6c14746 100644 --- a/model/Conference.php +++ b/model/Conference.php @@ -221,4 +221,8 @@ class Conference extends ModelBase public function getRelive() { return new Relive($this); } + + public function getExtraFiles() { + return $this->get('EXTRA_FILES', []); + } } diff --git a/model/Relive.php b/model/Relive.php index 7f0e747..5676b62 100644 --- a/model/Relive.php +++ b/model/Relive.php @@ -24,6 +24,11 @@ class Relive return $this->getConference()->get('CONFERENCE.RELIVE_JSON'); } + public function getJsonCache() + { + return sprintf('/tmp/relive-cache-%s', $this->getConference()->getSlug()); + } + public function getTalks() { if(!file_exists($this->getJsonUrl())) diff --git a/model/Schedule.php b/model/Schedule.php index eb9fe05..4032ecb 100644 --- a/model/Schedule.php +++ b/model/Schedule.php @@ -265,11 +265,16 @@ class Schedule return ((int)$parts[0] * 60 + (int)$parts[1]) * 60; } - private function getScheduleUrl() + public function getScheduleUrl() { return $this->getConference()->get('SCHEDULE.URL'); } + public function getScheduleCache() + { + return sprintf('/tmp/schedule-cache-%s', $this->getConference()->getSlug()); + } + public function getScheduleToRoomSlugMapping() { $mapping = array(); -- cgit v1.3.1 From 3384e963da66e9089b71c546750c8981be8c6ed8 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:11:47 +0100 Subject: change storage paths according to issue description --- command/download.php | 13 +++++-------- index.php | 3 +++ model/Relive.php | 2 +- model/Schedule.php | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'command') diff --git a/command/download.php b/command/download.php index 8deb454..17ae7c9 100644 --- a/command/download.php +++ b/command/download.php @@ -77,13 +77,13 @@ foreach ($conferences as $conference) ); } - foreach($conference->getExtraFiles() as $file) + foreach($conference->getExtraFiles() as $filename => $url) { download( 'extra-file', $conference, - $file, - get_file_cache($conference, $file) + $url, + get_file_cache($conference, $filename) ); } } @@ -91,12 +91,9 @@ foreach ($conferences as $conference) -function get_file_cache($conference, $url) +function get_file_cache($conference, $filename) { - $info = parse_url($url); - $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_'); - $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_'); - return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url)); + return joinpath([$GLOBALS['BASEDIR'], 'configs/conferences', $conference->getSlug(), $filename]); } function download($what, $conference, $url, $cache) diff --git a/index.php b/index.php index 9b4689c..ca0a6d4 100644 --- a/index.php +++ b/index.php @@ -3,6 +3,9 @@ if(!ini_get('short_open_tag')) die("`short_open_tag = On` is required\n"); +$GLOBALS['BASEDIR'] = dirname(__FILE__); +chdir($GLOBALS['BASEDIR']); + require_once('config.php'); require_once('lib/helper.php'); diff --git a/model/Relive.php b/model/Relive.php index 5676b62..1e8266a 100644 --- a/model/Relive.php +++ b/model/Relive.php @@ -26,7 +26,7 @@ class Relive public function getJsonCache() { - return sprintf('/tmp/relive-cache-%s', $this->getConference()->getSlug()); + return sprintf('/tmp/relive-cache-%s.json', $this->getConference()->getSlug()); } public function getTalks() diff --git a/model/Schedule.php b/model/Schedule.php index 4032ecb..6536864 100644 --- a/model/Schedule.php +++ b/model/Schedule.php @@ -272,7 +272,7 @@ class Schedule public function getScheduleCache() { - return sprintf('/tmp/schedule-cache-%s', $this->getConference()->getSlug()); + return sprintf('/tmp/schedule-cache-%s.xml', $this->getConference()->getSlug()); } public function getScheduleToRoomSlugMapping() -- cgit v1.3.1 From f4b2cb58b2e62e365103cbcffc3d8c2999b3f300 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:12:36 +0100 Subject: warn on old-style paths --- command/download.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'command') diff --git a/command/download.php b/command/download.php index 17ae7c9..c9fc5cd 100644 --- a/command/download.php +++ b/command/download.php @@ -98,6 +98,18 @@ function get_file_cache($conference, $filename) function download($what, $conference, $url, $cache) { + $info = parse_url($url); + if(!isset($info['scheme']) || !isset($info['host'])) + { + stderr( + ' !! %s url for conference %s does look like an old-style path: "%s". please update to a full http/https url', + $what, + $conference->getSlug(), + $url + ); + return false; + } + stdout( ' downloading %s from %s to %s', $what, -- cgit v1.3.1 From e7b0c27b54cdbcf248db191bcd34d5b8b4de2e99 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:12:56 +0100 Subject: improve log formatting --- command/download.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'command') diff --git a/command/download.php b/command/download.php index c9fc5cd..4018a9a 100644 --- a/command/download.php +++ b/command/download.php @@ -119,13 +119,14 @@ function download($what, $conference, $url, $cache) if(!do_download($url, $cache)) { stderr( - '!! download %s for conference %s from %s to %s failed miserably !!', + ' !! download %s for conference %s from %s to %s failed miserably !!', $what, $conference->getSlug(), $url, $cache ); } + return true; } function do_download($url, $cache) -- cgit v1.3.1 From 63967f91dacca321a63d91be4dd76d0ed3eb4956 Mon Sep 17 00:00:00 2001 From: MaZderMind Date: Sat, 17 Dec 2016 14:13:14 +0100 Subject: implement actual downloading --- command/download.php | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'command') diff --git a/command/download.php b/command/download.php index 4018a9a..486c7af 100644 --- a/command/download.php +++ b/command/download.php @@ -131,5 +131,33 @@ function download($what, $conference, $url, $cache) function do_download($url, $cache) { + $handle = curl_init($url); + curl_setopt_array($handle, [ + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_MAXREDIRS => 10, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_SSL_VERIFYPEER => false, /* accept all certificates, even self-signed */ + CURLOPT_SSL_VERIFYHOST => 2, /* verify hostname is in cert */ + CURLOPT_CONNECTTIMEOUT => 3, /* connect-timeout in seconds */ + CURLOPT_TIMEOUT => 5, /* transfer timeout im seconds */ + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, + CURLOPT_REFERER => 'https://streaming.media.ccc.de/', + CURLOPT_USERAGENT => '@c3voc Streaming-Website Downloader-Cronjob, Contact voc AT c3voc DOT de in case of problems. Might the Winkekatze be with you', + ]); + + $return = curl_exec($handle); + $info = curl_getinfo($handle); + curl_close($handle); + + if($info['http_code'] != 200) + return false; + + $tempfile = tempnam(dirname($cache), 'dl-'); + if(false === file_put_contents($tempfile, $return)) + return false; + + chmod($tempfile, 0644); + rename($tempfile, $cache); + return true; } -- cgit v1.3.1