From 4e85b423354d8b715d12540fbd2ac79a66de3399 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sun, 11 Dec 2016 19:56:22 +0100
Subject: add download-command cli interface & config
---
command/download.php | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
create mode 100644 command/download.php
(limited to 'command')
diff --git a/command/download.php b/command/download.php
new file mode 100644
index 0000000..8cc2f53
--- /dev/null
+++ b/command/download.php
@@ -0,0 +1,28 @@
+<?php
+
+$conf = $GLOBALS['CONFIG']['DOWNLOAD'];
+
+if(isset($conf['REQUIRE_USER']))
+{
+ if(get_current_user() != $conf['require-user'])
+ {
+ stderr(
+ 'Not downloading files for user %s, run this script as user %s',
+ get_current_user(),
+ $conf['require-user']
+ );
+ exit(2);
+ }
+}
+
+foreach (Conferences::getConferences() as $conference)
+{
+ stdout('== %s ==', $conference->getSlug());
+
+ if(isset($conf['MAX_CONFERENCE_AGE']))
+ {
+ date_diff()
+ return time() >= $this->endsAt();
+ }
+
+}
--
cgit v1.2.3
From 8beec6fda376b842a9ddb7a2425829690f0a8b48 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sun, 11 Dec 2016 20:27:35 +0100
Subject: use DateTime objects for ends/startsAt
---
command/download.php | 32 +++++++++++++++++++++++++-------
1 file changed, 25 insertions(+), 7 deletions(-)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index 8cc2f53..cb0987d 100644
--- a/command/download.php
+++ b/command/download.php
@@ -15,14 +15,32 @@ if(isset($conf['REQUIRE_USER']))
}
}
-foreach (Conferences::getConferences() as $conference)
+$conferences = Conferences::getConferences();
+
+if(isset($conf['MAX_CONFERENCE_AGE']))
{
- stdout('== %s ==', $conference->getSlug());
+ $months = intval($conf['MAX_CONFERENCE_AGE']);
+ $conferencesAfter = new DateTime();
+ $conferencesAfter->sub(new DateInterval('P'.$months.'D'));
- if(isset($conf['MAX_CONFERENCE_AGE']))
- {
- date_diff()
- return time() >= $this->endsAt();
- }
+ stdout('Filtering before %s', $conferencesAfter->format('Y-m-d'));
+ $conferences = array_filter($conferences, function($conference) use ($conferencesAfter) {
+ $isBefore = $conference->endsAt() < $conferencesAfter;
+
+ if($isBefore) {
+ stdout(
+ ' %s: %s',
+ $conference->endsAt()->format('Y-m-d'),
+ $conference->getSlug()
+ );
+ }
+ return !$isBefore;
+ });
+}
+
+stdout('');
+foreach ($conferences as $conference)
+{
+ stdout('== %s ==', $conference->getSlug());
}
--
cgit v1.2.3
From 6e0f7423769174b83d6b5618db7c33d54967099a Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sun, 11 Dec 2016 21:17:30 +0100
Subject: download files of forced-open conferences
---
command/download.php | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index cb0987d..d0999b4 100644
--- a/command/download.php
+++ b/command/download.php
@@ -23,8 +23,19 @@ if(isset($conf['MAX_CONFERENCE_AGE']))
$conferencesAfter = new DateTime();
$conferencesAfter->sub(new DateInterval('P'.$months.'D'));
- stdout('Filtering before %s', $conferencesAfter->format('Y-m-d'));
+ stdout('Skipping Conferences before %s', $conferencesAfter->format('Y-m-d'));
$conferences = array_filter($conferences, function($conference) use ($conferencesAfter) {
+ if($conference->isOpen())
+ {
+ stdout(
+ ' %s: %s',
+ '---open---',
+ $conference->getSlug()
+ );
+
+ return true;
+ }
+
$isBefore = $conference->endsAt() < $conferencesAfter;
if($isBefore) {
--
cgit v1.2.3
From 284878f0f9917b55359cd464b86508cacc472653 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sun, 11 Dec 2016 21:46:01 +0100
Subject: add cache-file getters and download preparations
---
command/download.php | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 68 insertions(+)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index d0999b4..8deb454 100644
--- a/command/download.php
+++ b/command/download.php
@@ -54,4 +54,72 @@ stdout('');
foreach ($conferences as $conference)
{
stdout('== %s ==', $conference->getSlug());
+
+ $relive = $conference->getRelive();
+ if($relive->isEnabled())
+ {
+ download(
+ 'relive-json',
+ $conference,
+ $relive->getJsonUrl(),
+ $relive->getJsonCache()
+ );
+ }
+
+ $schedule = $conference->getSchedule();
+ if($schedule->isEnabled())
+ {
+ download(
+ 'schedule-xml',
+ $conference,
+ $schedule->getScheduleUrl(),
+ $schedule->getScheduleCache()
+ );
+ }
+
+ foreach($conference->getExtraFiles() as $file)
+ {
+ download(
+ 'extra-file',
+ $conference,
+ $file,
+ get_file_cache($conference, $file)
+ );
+ }
+}
+
+
+
+
+function get_file_cache($conference, $url)
+{
+ $info = parse_url($url);
+ $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_');
+ $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_');
+ return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url));
+}
+
+function download($what, $conference, $url, $cache)
+{
+ stdout(
+ ' downloading %s from %s to %s',
+ $what,
+ $url,
+ $cache
+ );
+ if(!do_download($url, $cache))
+ {
+ stderr(
+ '!! download %s for conference %s from %s to %s failed miserably !!',
+ $what,
+ $conference->getSlug(),
+ $url,
+ $cache
+ );
+ }
+}
+
+function do_download($url, $cache)
+{
+ return true;
}
--
cgit v1.2.3
From 3384e963da66e9089b71c546750c8981be8c6ed8 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sat, 17 Dec 2016 14:11:47 +0100
Subject: change storage paths according to issue description
---
command/download.php | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index 8deb454..17ae7c9 100644
--- a/command/download.php
+++ b/command/download.php
@@ -77,13 +77,13 @@ foreach ($conferences as $conference)
);
}
- foreach($conference->getExtraFiles() as $file)
+ foreach($conference->getExtraFiles() as $filename => $url)
{
download(
'extra-file',
$conference,
- $file,
- get_file_cache($conference, $file)
+ $url,
+ get_file_cache($conference, $filename)
);
}
}
@@ -91,12 +91,9 @@ foreach ($conferences as $conference)
-function get_file_cache($conference, $url)
+function get_file_cache($conference, $filename)
{
- $info = parse_url($url);
- $host = trim(preg_replace('/[^a-z0-9]/i', '_', $info['host']), '_');
- $path = trim(preg_replace('/[^a-z0-9]/i', '_', $info['path']), '_');
- return sprintf('/tmp/file-cache-%s_%s_%s-%s', $conference->getSlug(), $host, $path, md5($url));
+ return joinpath([$GLOBALS['BASEDIR'], 'configs/conferences', $conference->getSlug(), $filename]);
}
function download($what, $conference, $url, $cache)
--
cgit v1.2.3
From f4b2cb58b2e62e365103cbcffc3d8c2999b3f300 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sat, 17 Dec 2016 14:12:36 +0100
Subject: warn on old-style paths
---
command/download.php | 12 ++++++++++++
1 file changed, 12 insertions(+)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index 17ae7c9..c9fc5cd 100644
--- a/command/download.php
+++ b/command/download.php
@@ -98,6 +98,18 @@ function get_file_cache($conference, $filename)
function download($what, $conference, $url, $cache)
{
+ $info = parse_url($url);
+ if(!isset($info['scheme']) || !isset($info['host']))
+ {
+ stderr(
+ ' !! %s url for conference %s does look like an old-style path: "%s". please update to a full http/https url',
+ $what,
+ $conference->getSlug(),
+ $url
+ );
+ return false;
+ }
+
stdout(
' downloading %s from %s to %s',
$what,
--
cgit v1.2.3
From e7b0c27b54cdbcf248db191bcd34d5b8b4de2e99 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sat, 17 Dec 2016 14:12:56 +0100
Subject: improve log formatting
---
command/download.php | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index c9fc5cd..4018a9a 100644
--- a/command/download.php
+++ b/command/download.php
@@ -119,13 +119,14 @@ function download($what, $conference, $url, $cache)
if(!do_download($url, $cache))
{
stderr(
- '!! download %s for conference %s from %s to %s failed miserably !!',
+ ' !! download %s for conference %s from %s to %s failed miserably !!',
$what,
$conference->getSlug(),
$url,
$cache
);
}
+ return true;
}
function do_download($url, $cache)
--
cgit v1.2.3
From 63967f91dacca321a63d91be4dd76d0ed3eb4956 Mon Sep 17 00:00:00 2001
From: MaZderMind
Date: Sat, 17 Dec 2016 14:13:14 +0100
Subject: implement actual downloading
---
command/download.php | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
(limited to 'command')
diff --git a/command/download.php b/command/download.php
index 4018a9a..486c7af 100644
--- a/command/download.php
+++ b/command/download.php
@@ -131,5 +131,33 @@ function download($what, $conference, $url, $cache)
function do_download($url, $cache)
{
+ $handle = curl_init($url);
+ curl_setopt_array($handle, [
+ CURLOPT_FOLLOWLOCATION => true,
+ CURLOPT_MAXREDIRS => 10,
+ CURLOPT_RETURNTRANSFER => true,
+ CURLOPT_SSL_VERIFYPEER => false, /* accept all certificates, even self-signed */
+ CURLOPT_SSL_VERIFYHOST => 2, /* verify hostname is in cert */
+ CURLOPT_CONNECTTIMEOUT => 3, /* connect-timeout in seconds */
+ CURLOPT_TIMEOUT => 5, /* transfer timeout im seconds */
+ CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
+ CURLOPT_REFERER => 'https://streaming.media.ccc.de/',
+ CURLOPT_USERAGENT => '@c3voc Streaming-Website Downloader-Cronjob, Contact voc AT c3voc DOT de in case of problems. Might the Winkekatze be with you',
+ ]);
+
+ $return = curl_exec($handle);
+ $info = curl_getinfo($handle);
+ curl_close($handle);
+
+ if($info['http_code'] != 200)
+ return false;
+
+ $tempfile = tempnam(dirname($cache), 'dl-');
+ if(false === file_put_contents($tempfile, $return))
+ return false;
+
+ chmod($tempfile, 0644);
+ rename($tempfile, $cache);
+
return true;
}
--
cgit v1.2.3