/**
* This command processes the Data pulled from twitter
*/
public function actionProcessData()
{
$CurrentDate = new \DateTime('NOW');
$crawlerDataCollection = CrawlerData::findAll(['parsed_at' => null]);
$json_data = [];
$regex = [];
foreach ($crawlerDataCollection as $crawlerData) {
$Contest = Contest::findOne($crawlerData->contest_id);
$contestRegex = CrawlerProfile::findOne(['id' => $Contest->crawler_profile_id]);
if ($Contest->custom_regex_entry == null) {
$regex["entry"] = $contestRegex->regex_entry;
} else {
$regex["entry"] = $Contest->custom_regex_entry;
}
if ($Contest->custom_regex_rating == null) {
$regex["rating"] = $contestRegex->regex_rating;
} else {
$regex["rating"] = $Contest->custom_regex_rating;
}
$jsonData = Json::decode($crawlerData->data, true);
foreach ($jsonData["statuses"] as $id => $tweet) {
// Get tweet creation date
$TweetDate = DateTime::createFromFormat("D M d H:i:s T Y", $tweet["created_at"]);
// check if Tweet creation date is within restrictions!
if ($TweetDate->format('Y-m-d') >= $Contest->parse_from && $TweetDate->format('Y-m-d') <= $Contest->parse_to) {
$Tweet = $this->CreateTweetEntry($tweet, $Contest->id, $regex);
if ($Tweet != null) {
$Contest->last_parsed_tweet_id = $Tweet->id;
}
}
}
if (array_key_exists("next_results", $jsonData["search_metadata"])) {
$Contest->next_result_query = $jsonData["search_metadata"]["next_results"];
} else {
// Set new refresh URL only if we have processed new tweets
if (sizeof($jsonData["statuses"]) > 0) {
$Contest->next_result_query = $jsonData["search_metadata"]["refresh_url"];
}
}
if ($Contest->save()) {
$this->stdout("Last Tweet ID for Contest " . $Contest->name . " saved!\n", Console::FG_GREEN);
// Update Entries with new Data
$this->UpdateContestEntries($Contest);
} else {
$this->stdout("Error Saving Tweet !\n", Console::BOLD);
$this->stdout(print_r($Contest->errors) . "\n", Console::BOLD);
}
$parsed_at = new DateTime('NOW');
$crawlerData->parsed_at = $parsed_at->format('Y-m-d H:i:s');
//save parsed_at time
$crawlerData->save();
}
}
<?php
echo $form->field($model, 'active')->widget(SwitchInput::classname(), []);
?>
<?php
echo $form->field($model, 'parse_from')->widget(DatePicker::classname(), ['options' => ['placeholder' => 'Enter Date where to start parsing ...'], 'pluginOptions' => ['autoclose' => true, 'format' => 'yyyy-mm-dd']]);
?>
<?php
echo $form->field($model, 'parse_to')->widget(DatePicker::classname(), ['options' => ['placeholder' => 'Enter Date where to end parsing ...'], 'pluginOptions' => ['autoclose' => true, 'format' => 'yyyy-mm-dd']]);
?>
<?php
echo $form->field($model, 'crawler_profile_id')->widget(Select2::classname(), ['data' => ArrayHelper::map(CrawlerProfile::find()->all(), 'id', 'name'), 'options' => ['placeholder' => 'Select a Crawler Profile ...'], 'pluginOptions' => ['allowClear' => true]]);
?>
<?php
echo $form->field($model, 'custom_regex_entry')->textarea(['rows' => 6]);
?>
<?php
echo $form->field($model, 'custom_regex_rating')->textarea(['rows' => 6]);
?>
<div class="form-group">
<?php
echo Html::submitButton($model->isNewRecord ? Yii::t('app', 'Create') : Yii::t('app', 'Update'), ['class' => $model->isNewRecord ? 'btn btn-success' : 'btn btn-primary']);
?>
</div>
