<?php
include __DIR__ . '/../SheetHubTool.php';
$list = array('土石流潛勢溪流圖層' => array('period' => 30 * 86400), '土壤圖' => array('period' => 30 * 86400), '山坡地範圍' => array('period' => 30 * 86400), '土石流潛勢溪流影響範圍圖層' => array('period' => 30 * 86400), '雨量站基本資料' => array('period' => 86400), '觀測站基本資料' => array('period' => 86400), '土石流紅黃警戒' => array('period' => 3600), '避難處所' => array('period' => 3600), '即時雨量' => array('period' => 600), '土石流警戒雨量' => array('period' => 3600), '醫療處所' => array('period' => 3600), '災例最速報' => array('period' => 3600), '觀測站影像資料' => array('period' => 3600), '災情查報單資料' => array('period' => 3600), '應變小組開設資料' => array('period' => 3600), '歷年土砂災害照片資料' => array('period' => 3600));
foreach ($list as $type => $config) {
$sheet_info = SheetHubTool::getSheetHubInfo('246.swcb.gov.tw', $type)->sheet;
$config['period'] = array_key_exists('period', $config) ? $config['period'] : 86400;
if (time() - strtotime($sheet_info->meta->fetched_time) < $config['period']) {
continue;
}
$config['source'] = $sheet_info->meta->source;
if (property_exists($sheet_info->meta, 'tab_id')) {
$config['tab_id'] = $sheet_info->meta->tab_id;
}
if (property_exists($sheet_info->meta, 'srs')) {
$config['srs'] = $sheet_info->meta->srs;
}
if (property_exists($sheet_info->meta, 'encode')) {
$config['encode'] = $sheet_info->meta->encode;
} elseif (!array_key_exists('encode', $config)) {
$config['encode'] = 'auto';
}
if (!$config['source']) {
error_log("找不到更新網址");
continue;
}
list($fp, $filetype) = SheetHubTool::downloadFile($config['source'], $sheet_info->meta->file_type);
$file = stream_get_meta_data($fp)['uri'];
$md5 = md5_file($file);
if ($sheet_info and $sheet_info->meta->file_hash and $md5 == $sheet_info->meta->file_hash) {
error_log("md5 same, skip {$type}");
<?php
include __DIR__ . '/../SheetHubTool.php';
$list = array('1998開始歷年每日登革熱確定病例統計' => array('period' => 3600 * 6));
foreach ($list as $type => $config) {
$sheet_info = SheetHubTool::getSheetHubInfo('ronnywang', $type)->sheet;
$config['period'] = array_key_exists('period', $config) ? $config['period'] : 86400;
if (time() - strtotime($sheet_info->meta->fetched_time) < $config['period']) {
continue;
}
$config['source'] = $sheet_info->meta->source;
if (!$config['source']) {
error_log("找不到更新網址");
continue;
}
list($fp, $filetype) = SheetHubTool::downloadFile($config['source'], $sheet_info->meta->file_type);
$file = stream_get_meta_data($fp)['uri'];
$md5 = md5_file($file);
if ($sheet_info and $sheet_info->meta->file_hash and $md5 == $sheet_info->meta->file_hash) {
error_log("md5 same, skip {$type}");
SheetHubTool::setMeta('ronnywang', $type, array('fetched_time' => date('c', time())));
continue;
}
$upload_id = SheetHubTool::uploadToSheetHub($fp, $filetype);
$ret = SheetHubTool::updateFile('ronnywang', $type, $upload_id, $config);
$result = " insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete);
error_log("Type={$type} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
$new_meta = $sheet_info->meta;
$new_meta->fetched_time = date('c', time());
$new_meta->file_hash = $md5;
SheetHubTool::setMeta('ronnywang', $type, $new_meta);
fputcsv($fp, $columns);
$column_added = true;
}
foreach ($values as $v) {
fputcsv($fp, $v);
}
}
$upload_id = SheetHubTool::uploadToSheetHub($fp, 'csv');
$ret = SheetHubTool::updateFile('opendata.cwb.gov.tw', '測站觀測資料', $upload_id);
$new_meta = $sheet_info->sheet->meta;
$new_meta->updated_time = date('Y/m/d', $data_time);
SheetHubTool::setMeta('opendata.cwb.gov.tw', '測站觀測資料', $new_meta);
}
// 每分鐘抓當下 15 分鐘資料
list($columns, $values) = CWB::getAll15mInfo();
$fp = tmpfile();
fputcsv($fp, $columns);
foreach ($values as $v) {
fputcsv($fp, $v);
}
$upload_id = SheetHubTool::uploadToSheetHub($fp, 'csv');
$sheet = '測站觀測資料_15分鐘_' . date('Y_m');
try {
$sheet_info = SheetHubTool::getSheetHubInfo('opendata.cwb.gov.tw', $sheet);
$ret = SheetHubTool::updateFile('opendata.cwb.gov.tw', $sheet, $upload_id);
error_log("done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
} catch (Exception $e) {
SheetHubTool::newFile('opendata.cwb.gov.tw', $sheet, $upload_id, array());
SheetHubTool::setURIColumns('opendata.cwb.gov.tw', $sheet, array(0, 2));
}
SheetHubTool::setMeta('opendata.cwb.gov.tw', $sheet, array('fetched_time' => date('Y/m/d H:i:s'), 'period' => '15分鐘', 'source' => 'http://www.cwb.gov.tw/V7/observe/real/ALLData.htm', 'update_code' => 'https://github.com/sheethub/data-import-script/blob/master/opendata.cwb.gov.tw/update.php'));
public function updateOrInsert($type, $config)
{
try {
if ($config['sheet_info']) {
$sheet_info = $config['sheet_info'];
} else {
$sheet_info = SheetHubTool::getSheetHubInfo('data.gov.tw', $type)->sheet;
}
$config['period'] = array_key_exists('period', $config) ? $config['period'] : 86400;
if (!$config['force'] and time() - strtotime($sheet_info->meta->fetched_time) < $config['period']) {
return "距離上次更新時間過短";
}
error_log("updating {$type}");
} catch (Exception $e) {
error_log("adding {$type} (because {$e->getMessage()}");
$sheet_info = null;
}
$time = microtime(true);
try {
if (!$config['source']) {
$config['source'] = $sheet_info->meta->source;
}
if (!$config['source']) {
throw new Exception("找不到更新網址");
}
$portal_meta = $this->getMetaFromPortal($config['source']);
} catch (Exception $e) {
$this->error($type, $e);
return "更新失敗，原因: " . $e->getMessage();
}
if (!$config['meta_only']) {
$files = array();
$download_url = null;
if (count($portal_meta['下載']) == 1) {
$download_url = $portal_meta['下載'][0]['url'];
$filetype = $portal_meta['下載'][0]['type'];
}
if (is_null($download_url) and $config['choose_file']) {
foreach ($portal_meta['下載'] as $info) {
if ($info['type'] == $config['choose_file']) {
$download_url = $info['url'];
$filetype = $info['type'];
break;
}
}
}
if (is_null($download_url)) {
$files = array();
foreach ($portal_meta['下載'] as $info) {
$files[$info['type']] = $info['url'];
}
foreach (array('json', 'csv', 'excel', 'xml') as $t) {
if ($files[$t]) {
$download_url = $files[$t];
$filetype = $t;
break;
}
}
}
if (is_null($download_url)) {
try {
throw new Exception("超過一個檔可以下載，不知道要用哪個: " . implode(',', array_map(function ($i) {
return $i['type'];
}, $portal_meta['下載'])));
} catch (Exception $e) {
$this->error($type, $e);
return "更新失敗，原因: " . $e->getMessage();
}
}
try {
error_log("downloading {$download_url}");
list($fp, $filetype) = SheetHubTool::downloadFile($download_url, $filetype);
error_log("downloaded");
} catch (Exception $e) {
$this->error($type, $e);
return "更新失敗，原因: " . $e->getMessage();
}
$file = stream_get_meta_data($fp)['uri'];
$md5 = md5_file($file);
if (!$config['force'] and $sheet_info and $sheet_info->meta->file_hash and $md5 == $sheet_info->meta->file_hash) {
error_log("md5 same, skip {$type}");
SheetHubTool::setMeta('data.gov.tw', $type, array('fetched_time' => date('c', time())));
return "下載原始檔案 md5 未變，不需更新";
}
if (property_exists($sheet_info->meta, 'updater:columns')) {
$config['columns'] = explode(',', $sheet_info->meta->{'updater:columns'});
}
if (property_exists($sheet_info->meta, 'updater:row_begin')) {
$config['row_begin'] = $sheet_info->meta->{'updater:row_begin'};
}
if ($config['filetype']) {
$filetype = $config['filetype'];
}
$fp = DataGovTw::specialCase($fp, $config['source']);
try {
$upload_id = SheetHubTool::uploadToSheetHub($fp, $filetype);
if ($sheet_info) {
$ret = SheetHubTool::updateFile('data.gov.tw', $type, $upload_id, $config);
$result = " insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete);
error_log("Type={$type} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
$this->updateStatus($type, date('c', time()), sprintf("更新完成，共新增 %d 筆，更新 %d 筆，刪除 %d 筆", count($ret->insert), count($ret->update), count($ret->delete)));
} else {
$sheet_info = SheetHubTool::newFile('data.gov.tw', $type, $upload_id, $config);
error_log("add {$type} done");
$this->updateStatus($type, date('c', time()), "建立完成");
}
} catch (Exception $e) {
$this->error($type, $e);
return "更新失敗，原因: " . $e->getMessage();
}
}
$new_meta = $sheet_info->meta;
if ($portal_meta['資料集修訂時間']) {
$new_meta->updated_time = $portal_meta['資料集修訂時間'];
}
$new_meta->fetched_time = date('c', time());
$new_meta->period = $portal_meta['更新頻率'];
$new_meta->source = $config['source'];
$new_meta->update_code = 'https://github.com/sheethub/data-import-script/blob/master/data.gov.tw/update.php';
$new_meta->license = $portal_meta['授權方式'] . ' ' . $portal_meta['授權說明網址'];
$new_meta->file_hash = $md5;
foreach (array('資料集提供機關', '資料集提供機關聯絡人', '資料集提供機關聯絡人電話', '備註') as $k) {
if ($portal_meta[$k]) {
$new_meta->{$k} = $portal_meta[$k];
}
}
SheetHubTool::setMeta('data.gov.tw', $type, $new_meta);
if ($portal_meta['資料集描述'] and $portal_meta['資料集描述'] != $sheet_info->description) {
SheetHubTool::setDescription('data.gov.tw', $type, $portal_meta['資料集描述']);
}
return "更新成功，已匯入 https://sheethub.com/data.gov.tw/{$type} , {$result}";
}
continue;
}
$description = $td_doms->item(3)->nodeValue;
if (!$description) {
continue;
}
$curl = curl_init("https://gist-map.motc.gov.tw" . $td_doms->item(5)->getElementsByTagName('a')->item(0)->getAttribute('href'));
curl_setopt($curl, CURLOPT_CAINFO, __DIR__ . '/GRCA2.crt');
$fp = tmpfile();
curl_setopt($curl, CURLOPT_FILE, $fp);
curl_exec($curl);
curl_close($curl);
$upload_id = SheetHubTool::uploadToSheetHub($fp, 'zip');
$new_file = false;
try {
$sheet_info = SheetHubTool::getSheetHubInfo('gist-map.motc.gov.tw', $title)->sheet;
} catch (Exception $e) {
$sheet_info = SheetHubTool::newFile('gist-map.motc.gov.tw', $title, $upload_id, array())->sheet;
$new_file = true;
}
if (!$new_file) {
$ret = SheetHubTool::updateFile('gist-map.motc.gov.tw', $title, $upload_id, array());
error_log("Type={$title} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
}
SheetHubTool::setDescription('gist-map.motc.gov.tw', $title, $description);
$file = stream_get_meta_data($fp)['uri'];
$md5 = md5_file($file);
$new_meta = $sheet_info->meta;
$new_meta->file_hash = $md5;
$new_meta->update_code = 'https://github.com/sheethub/data-import-script/blob/master/gist-map.motc.gov.tw/update.php';
$new_meta->source = 'https://gist-map.motc.gov.tw/Complex/MapTopic';