public function actionTrainData2() { $titleFile = fopen('titles.txt', 'w'); // Generate titles $titles = $this->getTitleRecord(); array_walk($titles, function ($title) use($titleFile) { fwrite($titleFile, $title['title'] . PHP_EOL); }); fclose($titleFile); echo 'Titles successfully generated.' . PHP_EOL; // Run sally command $sallyOptions = implode(' ', ['--input_format lines', '--output_format text', '--ngram_len 1', '--granularity tokens', '--vect_embed tfidf', '--vect_norm none', '--token_delim "%0a%0d%20%22.,:;!?"']); $sallyCommand = "sally {$sallyOptions} titles.txt titles.features.txt"; echo "Run: {$sallyCommand}\n"; exec($sallyCommand); // Read sally output and generate train set $trainFile = fopen('titles.train.svm', 'w'); $testFile = fopen('titles.test.svm', 'w'); $featuresFile = fopen('titles.features.txt', 'r'); $i = 0; $testIds = []; while (($line = fgets($featuresFile)) !== false) { $line = trim($line); if ($line[0] == '#') { continue; } // remove comment $posComment = strpos($line, '#'); $rest = trim(substr($line, 0, -(strlen($line) - $posComment))); if ($rest) { $featureTokens = explode(',', $rest); $features = []; foreach ($featureTokens as $token) { list($dimension, $weight) = explode('::', $token); $features[$dimension] = (double) $weight; } // find record /** @var Record $record */ $record = Record::findOne(['id' => $titles[$i]['id']]); $i++; $out = $testFile; $label = 0; $tasks = $record->getRecordTasks()->andWhere('is_prediction = 0')->all(); if ($tasks) { $out = $trainFile; $label = $tasks[0]->task_id; } else { $testIds[] = $record->id; } fwrite($out, $label . ' '); $parts = ['1:' . $record->window_id, '2:' . $record->window->process_id]; foreach ($features as $dim => $weight) { $parts[] = $dim . ':' . $weight; } fwrite($out, implode(' ', $parts)); fwrite($out, PHP_EOL); } } fclose($trainFile); fclose($testFile); // train libsvm model // best c=0.5, g=0.0078125 // best c=128.0, g=0.001953125, rate=67.3947 // best c=0.5, g=0.0078125, rate=99.5342 echo 'Run svm-train' . PHP_EOL; $trainCommnad = '/home/alx/soft/libsvm-3.20/svm-train -b 1 -c "0.5" -g "0.0078125" titles.train.svm svm.model'; exec($trainCommnad); // predict echo 'Run svm-predict' . PHP_EOL; $predictCommand = '/home/alx/soft/libsvm-3.20/svm-predict -b 1 -q titles.test.svm svm.model title.predict.txt'; exec($predictCommand); $predictionFile = fopen('title.predict.txt', 'r'); $i = 0; $transaction = \Yii::$app->db->beginTransaction(); // skip first line $dummyLine = fgets($predictionFile); $skiped = 0; while (($line = fgets($predictionFile)) !== false) { $line = trim($line); if (!isset($testIds[$i])) { throw new Exception('Что-то не сходится!'); } $recordId = $testIds[$i]; $i++; // Remove all predictions for this record RecordTask::deleteAll(['record_id' => $recordId, 'is_prediction' => 1]); $parts = explode(' ', $line); $predictTask = (int) $parts[0]; $probability = array_slice($parts, 1); $max = max(array_map('floatval', $probability)); if ($max < 0.75) { // echo "skip (with {$max})".PHP_EOL; $skiped++; continue; } // Save new prediction $model = new RecordTask(); $model->task_id = $predictTask; $model->record_id = $recordId; $model->is_prediction = 1; $model->save(false); } $transaction->commit(); echo "Skiped {$skiped} records\n"; }
public function afterDelete() { RecordTask::deleteAll(['task_id' => $this->id]); parent::afterDelete(); }
public function actionAssign() { $taskId = Yii::$app->request->getBodyParam('task'); $windowId = Yii::$app->request->getBodyParam('window'); $processId = Yii::$app->request->getBodyParam('process'); $transaction = Yii::$app->db->beginTransaction(); $timezone = new \DateTimeZone(Yii::$app->timeZone); $from = new \DateTime('today', $timezone); $to = new \DateTime('tomorrow', $timezone); try { $ids = Record::find()->with('window')->select(['id'])->andFilterWhere(['window_id' => $windowId, 'window.process_id' => $processId])->andWhere(['>=', 'start', $from->format('Y-m-d H:i:s')])->andWhere(['<', 'end', $to->format('Y-m-d H:i:s')])->createCommand()->queryColumn(); RecordTask::deleteAll(['record_id' => $ids]); $values = array_map(function ($id) use($taskId) { return [$id, $taskId]; }, $ids); Yii::$app->db->createCommand()->batchInsert('{{record_task}}', ['record_id', 'task_id'], $values)->execute(); $transaction->commit(); } catch (Exception $e) { $transaction->rollBack(); } }