/** * This command processes the Data pulled from twitter */ public function actionProcessData() { $CurrentDate = new \DateTime('NOW'); $crawlerDataCollection = CrawlerData::findAll(['parsed_at' => null]); $json_data = []; $regex = []; foreach ($crawlerDataCollection as $crawlerData) { $Contest = Contest::findOne($crawlerData->contest_id); $contestRegex = CrawlerProfile::findOne(['id' => $Contest->crawler_profile_id]); if ($Contest->custom_regex_entry == null) { $regex["entry"] = $contestRegex->regex_entry; } else { $regex["entry"] = $Contest->custom_regex_entry; } if ($Contest->custom_regex_rating == null) { $regex["rating"] = $contestRegex->regex_rating; } else { $regex["rating"] = $Contest->custom_regex_rating; } $jsonData = Json::decode($crawlerData->data, true); foreach ($jsonData["statuses"] as $id => $tweet) { // Get tweet creation date $TweetDate = DateTime::createFromFormat("D M d H:i:s T Y", $tweet["created_at"]); // check if Tweet creation date is within restrictions! if ($TweetDate->format('Y-m-d') >= $Contest->parse_from && $TweetDate->format('Y-m-d') <= $Contest->parse_to) { $Tweet = $this->CreateTweetEntry($tweet, $Contest->id, $regex); if ($Tweet != null) { $Contest->last_parsed_tweet_id = $Tweet->id; } } } if (array_key_exists("next_results", $jsonData["search_metadata"])) { $Contest->next_result_query = $jsonData["search_metadata"]["next_results"]; } else { // Set new refresh URL only if we have processed new tweets if (sizeof($jsonData["statuses"]) > 0) { $Contest->next_result_query = $jsonData["search_metadata"]["refresh_url"]; } } if ($Contest->save()) { $this->stdout("Last Tweet ID for Contest " . $Contest->name . " saved!\n", Console::FG_GREEN); // Update Entries with new Data $this->UpdateContestEntries($Contest); } else { $this->stdout("Error Saving Tweet !\n", Console::BOLD); $this->stdout(print_r($Contest->errors) . "\n", Console::BOLD); } $parsed_at = new DateTime('NOW'); $crawlerData->parsed_at = $parsed_at->format('Y-m-d H:i:s'); //save parsed_at time $crawlerData->save(); } }
<?php echo $form->field($model, 'active')->widget(SwitchInput::classname(), []); ?> <?php echo $form->field($model, 'parse_from')->widget(DatePicker::classname(), ['options' => ['placeholder' => 'Enter Date where to start parsing ...'], 'pluginOptions' => ['autoclose' => true, 'format' => 'yyyy-mm-dd']]); ?> <?php echo $form->field($model, 'parse_to')->widget(DatePicker::classname(), ['options' => ['placeholder' => 'Enter Date where to end parsing ...'], 'pluginOptions' => ['autoclose' => true, 'format' => 'yyyy-mm-dd']]); ?> <?php echo $form->field($model, 'crawler_profile_id')->widget(Select2::classname(), ['data' => ArrayHelper::map(CrawlerProfile::find()->all(), 'id', 'name'), 'options' => ['placeholder' => 'Select a Crawler Profile ...'], 'pluginOptions' => ['allowClear' => true]]); ?> <?php echo $form->field($model, 'custom_regex_entry')->textarea(['rows' => 6]); ?> <?php echo $form->field($model, 'custom_regex_rating')->textarea(['rows' => 6]); ?> <div class="form-group"> <?php echo Html::submitButton($model->isNewRecord ? Yii::t('app', 'Create') : Yii::t('app', 'Update'), ['class' => $model->isNewRecord ? 'btn btn-success' : 'btn btn-primary']); ?> </div>