Пример #1
0
        $config['tab_id'] = $sheet_info->meta->tab_id;
    }
    if (property_exists($sheet_info->meta, 'srs')) {
        $config['srs'] = $sheet_info->meta->srs;
    }
    if (property_exists($sheet_info->meta, 'encode')) {
        $config['encode'] = $sheet_info->meta->encode;
    } elseif (!array_key_exists('encode', $config)) {
        $config['encode'] = 'auto';
    }
    if (!$config['source']) {
        error_log("找不到更新網址");
        continue;
    }
    list($fp, $filetype) = SheetHubTool::downloadFile($config['source'], $sheet_info->meta->file_type);
    $file = stream_get_meta_data($fp)['uri'];
    $md5 = md5_file($file);
    if ($sheet_info and $sheet_info->meta->file_hash and $md5 == $sheet_info->meta->file_hash) {
        error_log("md5 same, skip {$type}");
        SheetHubTool::setMeta('246.swcb.gov.tw', $type, array('fetched_time' => date('c', time())));
        continue;
    }
    $upload_id = SheetHubTool::uploadToSheetHub($fp, $filetype);
    $ret = SheetHubTool::updateFile('246.swcb.gov.tw', $type, $upload_id, $config);
    $result = " insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete);
    error_log("Type={$type} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
    $new_meta = $sheet_info->meta;
    $new_meta->fetched_time = date('c', time());
    $new_meta->file_hash = $md5;
    SheetHubTool::setMeta('246.swcb.gov.tw', $type, $new_meta);
}
Пример #2
0
        }
        foreach ($town_names[$y][$values['TOWN_ID']] as $t_name) {
            foreach (explode("\n", $values['VILLAGE_NAME'] . "\n" . $values['VILLAGE_NAME_ALIAS']) as $n) {
                if (!$n) {
                    continue;
                }
                if (array_key_exists($t_name . $n, $used[$y])) {
                    if ($used[$y][$t_name . $n] == $values['VILLAGE_ID']) {
                        continue;
                    }
                    throw new Exception("{$t_name} . {$n} 同時對應到 {$values['VILLAGE_ID']} 和 {$used[$y][$t_name . $n]}");
                }
                $used[$y][$t_name . $n] = $values['VILLAGE_ID'];
                fputcsv(${'fp_' . $y}, array($t_name . $n, $values['VILLAGE_ID']));
            }
        }
    }
}
include 'SheetHubTool.php';
$id = SheetHubTool::uploadToSheetHub('result.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區', $id);
$id = SheetHubTool::uploadToSheetHub('result1984.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區_map_名稱1984', $id);
$id = SheetHubTool::uploadToSheetHub('result2010.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區_map_名稱2010', $id);
$id = SheetHubTool::uploadToSheetHub('result2014.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區_map_名稱2014', $id);
$id = SheetHubTool::uploadToSheetHub('map-village.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區_map_村里名稱', $id);
$id = SheetHubTool::uploadToSheetHub('map-town.csv', 'csv');
SheetHubTool::updateFile('area.reference.tw', '中華民國行政區_map_鄉鎮市區名稱', $id);
Пример #3
0
            fputcsv($fp, $columns);
            $column_added = true;
        }
        foreach ($values as $v) {
            fputcsv($fp, $v);
        }
    }
    $upload_id = SheetHubTool::uploadToSheetHub($fp, 'csv');
    $ret = SheetHubTool::updateFile('opendata.cwb.gov.tw', '測站觀測資料', $upload_id);
    $new_meta = $sheet_info->sheet->meta;
    $new_meta->updated_time = date('Y/m/d', $data_time);
    SheetHubTool::setMeta('opendata.cwb.gov.tw', '測站觀測資料', $new_meta);
}
// 每分鐘抓當下 15 分鐘資料
list($columns, $values) = CWB::getAll15mInfo();
$fp = tmpfile();
fputcsv($fp, $columns);
foreach ($values as $v) {
    fputcsv($fp, $v);
}
$upload_id = SheetHubTool::uploadToSheetHub($fp, 'csv');
$sheet = '測站觀測資料_15分鐘_' . date('Y_m');
try {
    $sheet_info = SheetHubTool::getSheetHubInfo('opendata.cwb.gov.tw', $sheet);
    $ret = SheetHubTool::updateFile('opendata.cwb.gov.tw', $sheet, $upload_id);
    error_log("done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
} catch (Exception $e) {
    SheetHubTool::newFile('opendata.cwb.gov.tw', $sheet, $upload_id, array());
    SheetHubTool::setURIColumns('opendata.cwb.gov.tw', $sheet, array(0, 2));
}
SheetHubTool::setMeta('opendata.cwb.gov.tw', $sheet, array('fetched_time' => date('Y/m/d H:i:s'), 'period' => '15分鐘', 'source' => 'http://www.cwb.gov.tw/V7/observe/real/ALLData.htm', 'update_code' => 'https://github.com/sheethub/data-import-script/blob/master/opendata.cwb.gov.tw/update.php'));
Пример #4
0
 public function updateOrInsert($type, $config)
 {
     try {
         if ($config['sheet_info']) {
             $sheet_info = $config['sheet_info'];
         } else {
             $sheet_info = SheetHubTool::getSheetHubInfo('data.gov.tw', $type)->sheet;
         }
         $config['period'] = array_key_exists('period', $config) ? $config['period'] : 86400;
         if (!$config['force'] and time() - strtotime($sheet_info->meta->fetched_time) < $config['period']) {
             return "距離上次更新時間過短";
         }
         error_log("updating {$type}");
     } catch (Exception $e) {
         error_log("adding {$type} (because {$e->getMessage()}");
         $sheet_info = null;
     }
     $time = microtime(true);
     try {
         if (!$config['source']) {
             $config['source'] = $sheet_info->meta->source;
         }
         if (!$config['source']) {
             throw new Exception("找不到更新網址");
         }
         $portal_meta = $this->getMetaFromPortal($config['source']);
     } catch (Exception $e) {
         $this->error($type, $e);
         return "更新失敗,原因: " . $e->getMessage();
     }
     if (!$config['meta_only']) {
         $files = array();
         $download_url = null;
         if (count($portal_meta['下載']) == 1) {
             $download_url = $portal_meta['下載'][0]['url'];
             $filetype = $portal_meta['下載'][0]['type'];
         }
         if (is_null($download_url) and $config['choose_file']) {
             foreach ($portal_meta['下載'] as $info) {
                 if ($info['type'] == $config['choose_file']) {
                     $download_url = $info['url'];
                     $filetype = $info['type'];
                     break;
                 }
             }
         }
         if (is_null($download_url)) {
             $files = array();
             foreach ($portal_meta['下載'] as $info) {
                 $files[$info['type']] = $info['url'];
             }
             foreach (array('json', 'csv', 'excel', 'xml') as $t) {
                 if ($files[$t]) {
                     $download_url = $files[$t];
                     $filetype = $t;
                     break;
                 }
             }
         }
         if (is_null($download_url)) {
             try {
                 throw new Exception("超過一個檔可以下載,不知道要用哪個: " . implode(',', array_map(function ($i) {
                     return $i['type'];
                 }, $portal_meta['下載'])));
             } catch (Exception $e) {
                 $this->error($type, $e);
                 return "更新失敗,原因: " . $e->getMessage();
             }
         }
         try {
             error_log("downloading {$download_url}");
             list($fp, $filetype) = SheetHubTool::downloadFile($download_url, $filetype);
             error_log("downloaded");
         } catch (Exception $e) {
             $this->error($type, $e);
             return "更新失敗,原因: " . $e->getMessage();
         }
         $file = stream_get_meta_data($fp)['uri'];
         $md5 = md5_file($file);
         if (!$config['force'] and $sheet_info and $sheet_info->meta->file_hash and $md5 == $sheet_info->meta->file_hash) {
             error_log("md5 same, skip {$type}");
             SheetHubTool::setMeta('data.gov.tw', $type, array('fetched_time' => date('c', time())));
             return "下載原始檔案 md5 未變,不需更新";
         }
         if (property_exists($sheet_info->meta, 'updater:columns')) {
             $config['columns'] = explode(',', $sheet_info->meta->{'updater:columns'});
         }
         if (property_exists($sheet_info->meta, 'updater:row_begin')) {
             $config['row_begin'] = $sheet_info->meta->{'updater:row_begin'};
         }
         if ($config['filetype']) {
             $filetype = $config['filetype'];
         }
         $fp = DataGovTw::specialCase($fp, $config['source']);
         try {
             $upload_id = SheetHubTool::uploadToSheetHub($fp, $filetype);
             if ($sheet_info) {
                 $ret = SheetHubTool::updateFile('data.gov.tw', $type, $upload_id, $config);
                 $result = " insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete);
                 error_log("Type={$type} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
                 $this->updateStatus($type, date('c', time()), sprintf("更新完成,共新增 %d 筆,更新 %d 筆,刪除 %d 筆", count($ret->insert), count($ret->update), count($ret->delete)));
             } else {
                 $sheet_info = SheetHubTool::newFile('data.gov.tw', $type, $upload_id, $config);
                 error_log("add {$type} done");
                 $this->updateStatus($type, date('c', time()), "建立完成");
             }
         } catch (Exception $e) {
             $this->error($type, $e);
             return "更新失敗,原因: " . $e->getMessage();
         }
     }
     $new_meta = $sheet_info->meta;
     if ($portal_meta['資料集修訂時間']) {
         $new_meta->updated_time = $portal_meta['資料集修訂時間'];
     }
     $new_meta->fetched_time = date('c', time());
     $new_meta->period = $portal_meta['更新頻率'];
     $new_meta->source = $config['source'];
     $new_meta->update_code = 'https://github.com/sheethub/data-import-script/blob/master/data.gov.tw/update.php';
     $new_meta->license = $portal_meta['授權方式'] . ' ' . $portal_meta['授權說明網址'];
     $new_meta->file_hash = $md5;
     foreach (array('資料集提供機關', '資料集提供機關聯絡人', '資料集提供機關聯絡人電話', '備註') as $k) {
         if ($portal_meta[$k]) {
             $new_meta->{$k} = $portal_meta[$k];
         }
     }
     SheetHubTool::setMeta('data.gov.tw', $type, $new_meta);
     if ($portal_meta['資料集描述'] and $portal_meta['資料集描述'] != $sheet_info->description) {
         SheetHubTool::setDescription('data.gov.tw', $type, $portal_meta['資料集描述']);
     }
     return "更新成功,已匯入 https://sheethub.com/data.gov.tw/{$type} , {$result}";
 }
Пример #5
0
 $td_doms = $tr_doms->item($i)->getElementsByTagName('td');
 $title = $td_doms->item(0)->nodeValue;
 if ($title == '國家風景區範圍') {
     continue;
 }
 $description = $td_doms->item(3)->nodeValue;
 if (!$description) {
     continue;
 }
 $curl = curl_init("https://gist-map.motc.gov.tw" . $td_doms->item(5)->getElementsByTagName('a')->item(0)->getAttribute('href'));
 curl_setopt($curl, CURLOPT_CAINFO, __DIR__ . '/GRCA2.crt');
 $fp = tmpfile();
 curl_setopt($curl, CURLOPT_FILE, $fp);
 curl_exec($curl);
 curl_close($curl);
 $upload_id = SheetHubTool::uploadToSheetHub($fp, 'zip');
 $new_file = false;
 try {
     $sheet_info = SheetHubTool::getSheetHubInfo('gist-map.motc.gov.tw', $title)->sheet;
 } catch (Exception $e) {
     $sheet_info = SheetHubTool::newFile('gist-map.motc.gov.tw', $title, $upload_id, array())->sheet;
     $new_file = true;
 }
 if (!$new_file) {
     $ret = SheetHubTool::updateFile('gist-map.motc.gov.tw', $title, $upload_id, array());
     error_log("Type={$title} done, insert: " . count($ret->insert) . ', update: ' . count($ret->update) . ', delete: ' . count($ret->delete));
 }
 SheetHubTool::setDescription('gist-map.motc.gov.tw', $title, $description);
 $file = stream_get_meta_data($fp)['uri'];
 $md5 = md5_file($file);
 $new_meta = $sheet_info->meta;