function scrap_yp($last_alphabet = '', $last_page = '')
{
$alphabet = range('a', 'z');
if (is_null($last_alphabet) || $last_alphabet == '') {
$temp_alphabet = scraperwiki::get_var('last_alphabet_loaded');
if (!is_null($temp_alphabet)) {
$last_alphabet = $temp_alphabet;
} else {
$last_alphabet = 'a';
}
}
if (is_null($last_page) || $last_page == '') {
$temp_page = scraperwiki::get_var('last_page_loaded');
if (!is_null($temp_page)) {
$last_page = $temp_page;
} else {
$last_page = 1;
}
}
$yp_base_url = 'http://www.yellowpages.co.id/browse/letter/' . $last_alphabet . '?page=' . $last_page;
$html = scraperWiki::scrape($yp_base_url);
$dom = new simple_html_dom();
$dom->load($html);
foreach ($dom->find("ul.directory-list") as $data) {
echo $data;
}
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Metasonix';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_metasonix'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Crumar';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_crumar'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Fairlight';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_fairlight'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Encore Electronics';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_encore_electronics'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
static function get_metadata($metadata_name, $default = null)
{
return scraperwiki::get_var($metadata_name, $default);
//return SW_MetadataClient::create()->get($metadata_name);
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Jen';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_jen'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Hammond';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_hammond'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'KeyFax Hardware';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_keyfax_hardware'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Propellerheads';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_propellerheads'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
<?php
require 'scraperwiki.php';
require 'scraperwiki/simple_html_dom.php';
$startProductId = scraperwiki::get_var("currentId", -1);
if ($startProductId == -1) {
print "No previous saved position found. Starting from scratch.";
} else {
print "Resuming from product id {$startProductId}\n";
}
scraperwiki::attach("hobbyking_batteryidlist");
$batteries = scraperwiki::select("id from hobbyking_batteryidlist.data where id > {$startProductId} order by id asc");
$remainingCount = count($batteries);
print "Found {$remainingCount} batteries left to be scraped.";
$maxPerRun = 100;
$loopCount = 0;
foreach ($batteries as $bat) {
if ($loopCount > $maxPerRun) {
print "Ending run after {$maxPerRun} iterations.";
break;
}
$productId = $bat['id'];
print "Retrieving " . $productId . "\n";
$html = scraperWiki::scrape("http://www.hobbyking.com/hobbyking/store/uh_viewItem.asp?idProduct={$productId}");
//print $html . "\n";
$dom = new simple_html_dom();
$dom->load($html);
// Get the product data (located in a span tag). Should only be one product data area!
$productDataAreasDom = $dom->find("SPAN[id=prodDataArea]");
$productDataDom = $productDataAreasDom[0];
//print $productData . "\n";
}
}
scraperwiki::save_sqlite(array('division_id','mp'),$votes,'vote');
} */
scraperwiki::save_var('last_id', $html['id']);
}
}
require 'scraperwiki/simple_html_dom.php';
//corrections:
//scraperwiki::save_var('last_id',55626); //55150
/*scraperwiki::sqliteexecute("delete from info where id>55652");
scraperwiki::sqlitecommit();
die();*/
//get last id
//scraperwiki::save_var('last_id',0);
$last_id = scraperwiki::get_var('last_id', 0);
echo $last_id;
//read the saved tables
scraperwiki::attach("cz_senate_voting_records_downloader_2", "src");
$rows = scraperwiki::select("id from src.swdata where id>{$last_id} order by id");
if (!empty($rows)) {
foreach ($rows as $html) {
//get dom
$dom = new simple_html_dom();
$html2 = scraperwiki::select("* from src.swdata where id={$html['id']}");
$dom->load(str_replace(" ", " ", $html2[0]['html']));
//common part
$div = $dom->find("div[class=wysiwyg]", 0);
//info
$h1 = $div->find('h1', 0);
preg_match('/([0-9]{1,}). schůze/', $h1->innertext, $matches);
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","Miscellaneous","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Teisco';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_teisco'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "Washington";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'CRB Elettronica';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_crb_elettronica'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Doepfer';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_doepfer'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
static function get_metadata($metadata_name, $default = null)
{
return scraperwiki::get_var($metadata_name, $default);
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Buchla';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_buchla'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","Miscellaneous","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Yamaha';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_yamaha'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
$noidung = $trr->find('div', 0)->innertext;
//$noidung = utf8_encode($noidung);
if (mb_strlen($noidung) > 1000) {
$j++;
@scraperwiki::save_sqlite(array('id'), array('id' => $j . '-' . $src[0]['url'], 'title' => $src[0]['title'], 'url' => $src[0]['url'], 'content' => base64_encode($noidung), 'order' => $j, 'num' => $src[0]['num'], 'reply' => $src[0]['reply']));
}
}
$html->clear();
unset($html);
scraperwiki::save_var('last_id', $i);
}
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach("s-in-s", "src");
//scraperwiki::save_var('last_id', 1);
//exit();
$id = scraperwiki::get_var('last_id');
for ($i = $id; $i < 1900; $i++) {
$src = scraperwiki::select("* from src.swdata limit {$i},1");
$url = $src[0]['link'];
$url = 'http://sexinsex.net/bbs/' . $url;
$html_content = scraperwiki::scrape($url);
$html = str_get_html($html_content);
$data = array();
$tr = $html->find("div.postmessage div.t_msgfont");
$j = 0;
foreach ($tr as $trr) {
$noidung = $trr->find('div', 0)->innertext;
//$noidung = utf8_encode($noidung);
if (mb_strlen($noidung) > 1000) {
$j++;
@scraperwiki::save_sqlite(array('id'), array('id' => $j . '-' . $src[0]['url'], 'title' => $src[0]['title'], 'url' => $src[0]['url'], 'content' => base64_encode($noidung), 'order' => $j, 'num' => $src[0]['num'], 'reply' => $src[0]['reply']));
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "Washington";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'M-Audio';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_maudio'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "Washington";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Advanced Tools for the Arts (ATA)';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_advanced_tools_for_the_arts'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","Miscellaneous","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Whitehall';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_whitehall'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Alesis';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_alesis'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
function searchForId($id, $array)
{
foreach ($array as $key => $val) {
if ($val['COTTAGE_URL'] === $id) {
return $key;
}
}
return null;
}
$blacklist = array();
$url = "http://www.coastandcountry.co.uk/cottage-details/";
scraperwiki::attach("coastandcountrycouk");
# get an array of the cottage data to scrape
$cottData = scraperwiki::select("COTTAGE_URL, PRICE_HIGH, PRICE_LOW from 'coastandcountrycouk'.SWDATA order by COTTAGE_URL");
$placeholder = scraperwiki::get_var("cottID");
if ($placeholder != "") {
$index = searchForId($placeholder, $cottData);
$cottData = array_splice($cottData, $index);
}
require 'scraperwiki/simple_html_dom.php';
$dom = new simple_html_dom();
foreach ($cottData as $value) {
scraperwiki::save_var("cottID", $value['COTTAGE_URL']);
// check the cottage url against the blacklist
foreach ($blacklist as $blItem) {
if ($value['COTTAGE_URL'] == $blItem) {
continue 2;
}
}
//load the page into the scraper
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = scraperwiki::get_var('state');
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Hartmann';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_hartmann'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
scraperwiki::save(array('id'), $dbdata);
$html->__destruct();
}
//End single room load
}
// End the listingnum loop
}
// End if
scraperwiki::save_var('last_page', $i);
$html->__destruct();
}
// End the for loop
require 'scraperwiki/simple_html_dom.php';
set_time_limit(0);
// While $i (the loop counter) is less than or equal to 9999 (the number of times you want to navigate)
$lastpage = scraperwiki::get_var('last_page');
//$lastpage = 60;
print $lastpage;
for ($i = $lastpage; $i <= 9999; $i++) {
$html_content = scraperwiki::scrape("https://www.airbnb.co.uk/s?page=" . $i . "&room_types[]=Entire+home%2Fapt");
$html = str_get_html($html_content);
if (!empty($html)) {
$listingnum = $html->find("li.search_result");
foreach ($listingnum as $el) {
//print $el-> . "\n";
$url_listing = array('listingnum' => $el->getAttribute('data-hosting-id'), 'neighborhood' => isset($el->find('span.neighborhood-link', 0)->plaintext) ? $el->find('span.neighborhood-link', 0)->plaintext : null, 'url' => 'https://www.airbnb.com/rooms/' . $el->getAttribute('data-hosting-id'));
$neigh = $url_listing['neighborhood'];
$hostingid = $url_listing['listingnum'];
$url_listing = $url_listing['url'];
$url_listing = scraperwiki::scrape($url_listing);
$html = str_get_html($url_listing);
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","Miscellaneous","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Stramp';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_stramp'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
border-radius: 15px;
max-width: 400px;
}
.tip ul { margin: 0; padding: 0; text-align:center; }
.tip ul li{ list-style-type: none; padding: 0; margin: 0; }
.tip ul li.synth_name { font-size: 20px; }
.tip ul li.price { font-size: 16px; padding: 5px; color: green; }
.tip ul li.date { font-size: 14px; color: #e7e7e7; text-align: right; }
.tip ul li.description { font-size: 14px; color: #e7e7e7; text-align: left; padding-bottom: 15px; }
</style>
<div id="content">
<h1 align='center'><a href="http://synthfilter.com" title="Synth Filter">SynthFilter</a>: <a href='http://geo.craigslist.org/iso/us/ca' target='_blank'>California</a> - Craigslist</h1>
<div class='synth-container'>
<?php
scraperwiki::attach('synthfilter_utils');
$manufacturers = scraperwiki::get_var('manufacturers');
$manufacturers = explode(',', $manufacturers);
//Convert to array
if (!empty($_GET['start'])) {
$start = $_GET['start'];
}
// To take care global variable if OFF
if (empty($start)) {
$start = 0;
}
if (strlen($start) > 0 and !is_numeric($start)) {
//echo "Data Error";
//exit;
$start = 0;
}
$sourcescraper = 'california_craigslist_synth_collector';
}
/*
"Ampron","Atlantex","ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Buchla","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "Washington";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'Digisound';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_digisound'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
}
/*
"ARP","Access Music","Akai","Alesis","Analogue Systems","Applied Acoustics","Aries","Arturia","BOSS","BitHeadz","Bomb Factory Studios","Casio","Chamberlin","Cheetah","Chimera Synthesis","Clavia","Con Brio","Creamware","Crumar","Dave Smith Instruments","Doepfer","E-mu Systems","EDP","EML","EMS","Electrix Pro","Electro Harmonix","Elektron","Elka","Encore Electronics","Ensoniq","FBT Electronica","Fairlight","Farfisa","Formanta","Future Retro","GForce Software","Generalmusic","Gleeman","Hammond","Hartmann","Hohner","IK Multimedia","Image Line","Jen Electronics","JoMoX","Kawai","Kenton Electronics","KeyFax Hardware","Koblo","Korg","Kurzweil","Linn Electronics","Logan Electronics","MAM","MOTU","MacBeth Studio Systems","Marion Systems","Metasonix","??Miscellaneous??","Moog Music","Mutronics","Native Instruments","New England Digital","Novation","OSC","Oberheim","Octave","PAiA","PPG","Propellerheads","Prosoniq","Quasimidi","Red Sound Systems","Rhodes","Roland"
*/
require 'scraperwiki/simple_html_dom.php';
scraperwiki::attach('synthfilter_utils');
$state = "California";
//Must be Titlecase e.g, Alabama, Michigan. or use an array separated by commas Alabama, Washington
$states = explode(",", $state);
print_r($states);
$jsonManufacturerScraper = scraperwiki::get_var('manufacturer_scraper');
//Get a unique list of synth manufacturers
$jsonManufacturerData = file_get_contents($jsonManufacturerScraper);
$manufacturerQuery = 'E-mu Systems';
//You can choose to search only a specific manufacturer or if not, just leave it blank
$ignoreWords = explode(',', scraperwiki::get_var('iw_e-mu_systems'));
echo "Total ignored words: " . count($ignoreWords) . "\n";
$cityDepth = 0;
//Set to 0 to search all cities found in the database
$synthDepth = 0;
//Set to 0 to search all synths found in the database
$manufacturerDepth = 0;
//Set to 0 to search all synth manufacturers found in the database
$depthOverride = true;
//Cancels the depth variables
//Delete existing data
$info = scraperwiki::table_info($name = "swdata");
if (!empty($info)) {
scraperwiki::sqliteexecute("DELETE FROM swdata");
//Truncate the table before adding new results
}
