Merge branch 'feature/20-declarative-downloading'

fixes #20

13 files changed, 271 insertions, 31 deletions

diff --git a/command/download.php b/command/download.php
new file mode 100644
index 0000000..486c7af
--- /dev/null
+++ b/command/download.php
@@ -0,0 +1,163 @@
+<?php
+
+$conf = $GLOBALS['CONFIG']['DOWNLOAD'];
+
+if(isset($conf['REQUIRE_USER']))
+{
+	if(get_current_user() != $conf['require-user'])
+	{
+		stderr(
+			'Not downloading files for user %s, run this script as user %s',
+			get_current_user(),
+			$conf['require-user']
+		);
+		exit(2);
+	}
+}
+
+$conferences = Conferences::getConferences();
+
+if(isset($conf['MAX_CONFERENCE_AGE']))
+{
+	$months = intval($conf['MAX_CONFERENCE_AGE']);
+	$conferencesAfter = new DateTime();
+	$conferencesAfter->sub(new DateInterval('P'.$months.'D'));
+
+	stdout('Skipping Conferences before %s', $conferencesAfter->format('Y-m-d'));
+	$conferences = array_filter($conferences, function($conference) use ($conferencesAfter) {
+		if($conference->isOpen())
+		{
+			stdout(
+				'  %s: %s',
+				'---open---',
+				$conference->getSlug()
+			);
+
+			return true;
+		}
+
+		$isBefore = $conference->endsAt() < $conferencesAfter;
+
+		if($isBefore) {
+			stdout(
+				'  %s: %s',
+				$conference->endsAt()->format('Y-m-d'),
+				$conference->getSlug()
+			);
+		}
+
+		return !$isBefore;
+	});
+}
+
+stdout('');
+foreach ($conferences as $conference)
+{
+	stdout('== %s ==', $conference->getSlug());
+
+	$relive = $conference->getRelive();
+	if($relive->isEnabled())
+	{
+		download(
+			'relive-json',
+			$conference,
+			$relive->getJsonUrl(),
+			$relive->getJsonCache()
+		);
+	}
+
+	$schedule = $conference->getSchedule();
+	if($schedule->isEnabled())
+	{
+		download(
+			'schedule-xml',
+			$conference,
+			$schedule->getScheduleUrl(),
+			$schedule->getScheduleCache()
+		);
+	}
+
+	foreach($conference->getExtraFiles() as $filename => $url)
+	{
+		download(
+			'extra-file',
+			$conference,
+			$url,
+			get_file_cache($conference, $filename)
+		);
+	}
+}
+
+
+
+
+function get_file_cache($conference, $filename)
+{
+	return joinpath([$GLOBALS['BASEDIR'], 'configs/conferences', $conference->getSlug(), $filename]);
+}
+
+function download($what, $conference, $url, $cache)
+{
+	$info = parse_url($url);
+	if(!isset($info['scheme']) || !isset($info['host']))
+	{
+		stderr(
+			'  !! %s url for conference %s does look like an old-style path: "%s". please update to a full http/https url',
+			$what,
+			$conference->getSlug(),
+			$url
+		);
+		return false;
+	}
+
+	stdout(
+		'  downloading %s from %s to %s',
+		$what,
+		$url,
+		$cache
+	);
+	if(!do_download($url, $cache))
+	{
+		stderr(
+			'  !! download %s for conference %s from %s to %s failed miserably !!',
+			$what,
+			$conference->getSlug(),
+			$url,
+			$cache
+		);
+	}
+	return true;
+}
+
+function do_download($url, $cache)
+{
+	$handle = curl_init($url);
+	curl_setopt_array($handle, [
+		CURLOPT_FOLLOWLOCATION  => true,
+		CURLOPT_MAXREDIRS       => 10,
+		CURLOPT_RETURNTRANSFER  => true,
+		CURLOPT_SSL_VERIFYPEER  => false, /* accept all certificates, even self-signed */
+		CURLOPT_SSL_VERIFYHOST  => 2,     /* verify hostname is in cert */
+		CURLOPT_CONNECTTIMEOUT  => 3,     /* connect-timeout in seconds */
+		CURLOPT_TIMEOUT         => 5,     /* transfer timeout im seconds */
+		CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
+		CURLOPT_REFERER         => 'https://streaming.media.ccc.de/',
+		CURLOPT_USERAGENT       => '@c3voc Streaming-Website Downloader-Cronjob, Contact voc AT c3voc DOT de in case of problems. Might the Winkekatze be with you',
+	]);
+
+	$return = curl_exec($handle);
+	$info = curl_getinfo($handle);
+	curl_close($handle);
+
+	if($info['http_code'] != 200)
+		return false;
+
+	$tempfile = tempnam(dirname($cache), 'dl-');
+	if(false === file_put_contents($tempfile, $return))
+		return false;
+
+	chmod($tempfile, 0644);
+	rename($tempfile, $cache);
+
+	return true;
+}
diff --git a/config.php b/config.php
index 1dbea27..1b5e84f 100644
--- a/config.php
+++ b/config.php
@@ -24,7 +24,7 @@ $GLOBALS['CONFIG']['PREVIEW_DOMAIN'] = 'xlocalhost';
  * Protokollfreie URLs (welche, die mit // beginnen), werden automatisch mit dem korrekten Protokoll ergänzt.
  * In diesem Fall wird auch ein SSL-Umschalt-Button im Header angezeigt
  */
-if($_SERVER['SERVER_NAME'] == 'localhost')
+if(@$_SERVER['SERVER_NAME'] == 'localhost')
 {
 	// keine Konfiguration -> BASEURL wird automatisch erraten
 }
@@ -38,3 +38,28 @@ else
 	// Set a safe Default
 	$GLOBALS['CONFIG']['BASEURL'] = '//streaming.media.ccc.de/';
 }
+
+
+/**
+ * Konfiguration für den Datei-Download Cronjob
+ */
+$GLOBALS['CONFIG']['DOWNLOAD'] = [
+	/**
+	 * Verweigeren Download, wenn der PHP-Prozess unter einem anderen Benutzer als diesem läuft
+	 * Auskommentieren um alle Benutzer zu erlauben
+	 */
+	//'REQUIRE_USER' => 'www-data',
+
+	/**
+	 * Wartende HTTP-Downloads nach dieser Anzahl von Sekunden abbrechen
+	 */
+	'HTTP_TIMEOUT' => 5 /* Sekunden */,
+
+	/**
+	 * Nur Dateien von Konferenzen herunterladen, die weniger als
+	 * diese Aanzahl von Tagen alt sind (gemessen am END_DATE)
+	 *
+	 * Auskommentieren, um alle Konferenzen zu beachten
+	 */
+	'MAX_CONFERENCE_AGE' => 14 /* Tage */,
+];
diff --git a/configs/conferences/32c3/config.php b/configs/conferences/32c3/config.php
index eda9450..f1cc5a6 100644
--- a/configs/conferences/32c3/config.php
+++ b/configs/conferences/32c3/config.php
@@ -667,5 +667,11 @@ $CONFIG['TWITTER'] = array(
 	'TEXT' => '#32C3 #%s',
 );
 
+$CONFIG['EXTRA_FILES'] = array(
+	'schedule.xml' => 'https://events.ccc.de/congress/2015/Fahrplan/schedule.xml',
+	'schedule.json' => 'https://events.ccc.de/congress/2015/Fahrplan/schedule.json',
+	'everything.schedule.xml' => 'http://data.testi.ber.c3voc.de/32C3/everything.schedule.xml',
+	'everything.schedule.json' => 'http://data.testi.ber.c3voc.de/32C3/everything.schedule.json',
+);
 
 return $CONFIG;
diff --git a/configs/conferences/chaosradio/config.php b/configs/conferences/chaosradio/config.php
index f776552..a435760 100644
--- a/configs/conferences/chaosradio/config.php
+++ b/configs/conferences/chaosradio/config.php
@@ -121,7 +121,7 @@ $CONFIG['CONFERENCE'] = array(
 	 *
 	 * Wird diese Zeile auskommentiert, wird der Link nicht angezeigt
 	 */
-	'RELIVE_JSON' => 'configs/conferences/chaosradio/vod.json',
+	'RELIVE_JSON' => 'http://live.dus.c3voc.de/relive/chaosradio/index.json',
 );
 
 /**
diff --git a/configs/conferences/chaosradio/download.sh b/configs/conferences/chaosradio/download.sh
deleted file mode 100644
index 5474658..0000000
--- a/configs/conferences/chaosradio/download.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-# vod json
-wget -q "http://live.dus.c3voc.de/relive/chaosradio/index.json" -O /tmp/vod.json && mv /tmp/vod.json vod.json
-rm -f /tmp/vod.json
diff --git a/configs/conferences/datengarten/config.php b/configs/conferences/datengarten/config.php
index e2e21eb..f8514be 100644
--- a/configs/conferences/datengarten/config.php
+++ b/configs/conferences/datengarten/config.php
@@ -117,8 +117,7 @@ $CONFIG['CONFERENCE'] = array(
 	 *
 	 * Wird diese Zeile auskommentiert, wird der Link nicht angezeigt
 	 */
-	//'RELIVE_JSON' => 'configs/vod.json',
-        'RELIVE_JSON' => 'configs/conferences/datengarten/vod.json',
+	'RELIVE_JSON' => 'http://live.dus.c3voc.de/relive/datengarten/index.json',
 );
 
 /**
diff --git a/configs/conferences/datengarten/download.sh b/configs/conferences/datengarten/download.sh
deleted file mode 100644
index 5834c4b..0000000
--- a/configs/conferences/datengarten/download.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-# vod json
-wget -q "http://live.dus.c3voc.de/relive/datengarten/index.json" -O /tmp/vod.json && mv /tmp/vod.json vod.json
-rm -f /tmp/vod.json
diff --git a/index.php b/index.php
index ef3944c..ca0a6d4 100644
--- a/index.php
+++ b/index.php
@@ -1,7 +1,10 @@
 <?php
 
 if(!ini_get('short_open_tag'))
-	die('`short_open_tag = On` is required');
+	die("`short_open_tag = On` is required\n");
+
+$GLOBALS['BASEDIR'] = dirname(__FILE__);
+chdir($GLOBALS['BASEDIR']);
 
 require_once('config.php');
 require_once('lib/helper.php');
@@ -27,6 +30,22 @@ require_once('model/Upcoming.php');
 
 
 ob_start();
+if(isset($argv) && isset($argv[1]))
+{
+	require('lib/command-helper.php');
+
+	switch($argv[1])
+	{
+		case 'download':
+			require('command/download.php');
+			exit(0);
+	}
+
+	stderr("Unknown Command: %s", $argv[1]);
+	exit(1);
+}
+
+
 try {
 	if(isset($_GET['htaccess']))
 	{
diff --git a/lib/command-helper.php b/lib/command-helper.php
new file mode 100644
index 0000000..c406b9e
--- /dev/null
+++ b/lib/command-helper.php
@@ -0,0 +1,15 @@
+<?php
+
+function stderr($str) {
+	$args = func_get_args();
+	$args[0] = $args[0]."\n";
+	array_unshift($args, STDERR);
+	call_user_func_array('fprintf', $args);
+}
+
+function stdout($str) {
+	$args = func_get_args();
+	$args[0] = $args[0]."\n";
+	array_unshift($args, STDOUT);
+	call_user_func_array('fprintf', $args);
+}
diff --git a/model/Conference.php b/model/Conference.php
index 07d72dd..8d387a0 100644
--- a/model/Conference.php
+++ b/model/Conference.php
@@ -19,7 +19,7 @@ class Conference extends ModelBase
 	}
 
 	public function isPreviewEnabled() {
-		if($GLOBALS['forceopen'])
+		if(@$GLOBALS['forceopen'])
 			return true;
 
 		if($this->has('PREVIEW_DOMAIN') && ($this->get('PREVIEW_DOMAIN') == $_SERVER['SERVER_NAME']))
@@ -32,12 +32,22 @@ class Conference extends ModelBase
 		return !$this->hasBegun() || $this->hasEnded();
 	}
 
+	public function isOpen() {
+		return !$this->isClosed();
+	}
+
 	public function startsAt() {
-		return $this->get('CONFERENCE.STARTS_AT');
+		if(!$this->has('CONFERENCE.STARTS_AT'))
+			return null;
+
+		return DateTime::createFromFormat('U', $this->get('CONFERENCE.STARTS_AT'));
 	}
 
 	public function endsAt() {
-		return $this->get('CONFERENCE.ENDS_AT');
+		if(!$this->has('CONFERENCE.ENDS_AT'))
+			return null;
+
+		return DateTime::createFromFormat('U', $this->get('CONFERENCE.ENDS_AT'));
 	}
 
 	public function hasBegun() {
@@ -62,7 +72,8 @@ class Conference extends ModelBase
 		}
 
 		if($this->has('CONFERENCE.STARTS_AT')) {
-			return time() >= $this->get('CONFERENCE.STARTS_AT');
+			$now = new DateTime('now');
+			return $now >= $this->startsAt();
 		} else {
 			return true;
 		}
@@ -84,7 +95,8 @@ class Conference extends ModelBase
 		}
 
 		if($this->has('CONFERENCE.ENDS_AT')) {
-			return time() >= $this->get('CONFERENCE.ENDS_AT');
+			$now = new DateTime('now');
+			return $now >= $this->endsAt();
 		} else {
 			return false;
 		}
@@ -128,10 +140,10 @@ class Conference extends ModelBase
 	}
 
 	public function hasRelive() {
-		return $this->has('CONFERENCE.RELIVE_JSON');
+		return $this->getRelive()->isEnabled();
 	}
 	public function getReliveUrl() {
-		if($this->has('CONFERENCE.RELIVE_JSON'))
+		if($this->getRelive()->isEnabled())
 			return joinpath([$this->getSlug(), 'relive']).url_params();
 
 		else
@@ -209,4 +221,8 @@ class Conference extends ModelBase
 	public function getRelive() {
 		return new Relive($this);
 	}
+
+	public function getExtraFiles() {
+		return $this->get('EXTRA_FILES', []);
+	}
 }
diff --git a/model/Conferences.php b/model/Conferences.php
index 2dc9f35..f930e32 100644
--- a/model/Conferences.php
+++ b/model/Conferences.php
@@ -33,7 +33,7 @@ class Conferences
 
 	public static function getActiveConferences() {
 		return array_values(array_filter(
-			Conferences::getConferences(),
+			Conferences::getConferencesSorted(),
 			function($conference) {
 				return !$conference->isClosed();
 			}
@@ -48,7 +48,7 @@ class Conferences
 		$sorted = Conferences::getConferences();
 
 		usort($sorted, function($a, $b) {
-			return $b->startsAt() - $a->endsAt();
+			return $b->startsAt() > $a->endsAt() ? 1 : -1;
 		});
 
 		return $sorted;
diff --git a/model/Relive.php b/model/Relive.php
index 7f0e747..542836d 100644
--- a/model/Relive.php
+++ b/model/Relive.php
@@ -24,9 +24,14 @@ class Relive
 		return $this->getConference()->get('CONFERENCE.RELIVE_JSON');
 	}
 
+	public function getJsonCache()
+	{
+		return sprintf('/tmp/relive-cache-%s.json', $this->getConference()->getSlug());
+	}
+
 	public function getTalks()
 	{
-		if(!file_exists($this->getJsonUrl()))
+		if(!file_exists($this->getJsonCache()))
 			return array();
 
 		$talks = file_get_contents($this->getJsonUrl());
diff --git a/model/Schedule.php b/model/Schedule.php
index eb9fe05..b54709e 100644
--- a/model/Schedule.php
+++ b/model/Schedule.php
@@ -36,17 +36,10 @@ class Schedule
 
 	private function fetchSchedule()
 	{
-		$opts = array(
-			'http' => array(
-				'timeout' => 2,
-				'user_agent' => 'C3VOC Universal Streaming-Website Backend @ '.$_SERVER['HTTP_HOST'],
-			)
-		);
-		$context  = stream_context_create($opts);
-		$schedule = file_get_contents($this->getScheduleUrl(), false, $context);
+		$schedule = file_get_contents($this->getScheduleCache());
 
 		if(!$schedule)
-			throw new ScheduleException("Error Downloading Schedule from ".$this->getConference()->getScheduleUrl());
+			throw new ScheduleException("Error Loading Schedule from ".$this->getScheduleCache());
 
 		return simplexml_load_string($schedule);
 	}
@@ -265,11 +258,16 @@ class Schedule
 		return ((int)$parts[0] * 60 + (int)$parts[1]) * 60;
 	}
 
-	private function getScheduleUrl()
+	public function getScheduleUrl()
 	{
 		return $this->getConference()->get('SCHEDULE.URL');
 	}
 
+	public function getScheduleCache()
+	{
+		return sprintf('/tmp/schedule-cache-%s.xml', $this->getConference()->getSlug());
+	}
+
 	public function getScheduleToRoomSlugMapping()
 	{
 		$mapping = array();