public function actionTrainData2()
{
$titleFile = fopen('titles.txt', 'w');
// Generate titles
$titles = $this->getTitleRecord();
array_walk($titles, function ($title) use($titleFile) {
fwrite($titleFile, $title['title'] . PHP_EOL);
});
fclose($titleFile);
echo 'Titles successfully generated.' . PHP_EOL;
// Run sally command
$sallyOptions = implode(' ', ['--input_format lines', '--output_format text', '--ngram_len 1', '--granularity tokens', '--vect_embed tfidf', '--vect_norm none', '--token_delim "%0a%0d%20%22.,:;!?"']);
$sallyCommand = "sally {$sallyOptions} titles.txt titles.features.txt";
echo "Run: {$sallyCommand}\n";
exec($sallyCommand);
// Read sally output and generate train set
$trainFile = fopen('titles.train.svm', 'w');
$testFile = fopen('titles.test.svm', 'w');
$featuresFile = fopen('titles.features.txt', 'r');
$i = 0;
$testIds = [];
while (($line = fgets($featuresFile)) !== false) {
$line = trim($line);
if ($line[0] == '#') {
continue;
}
// remove comment
$posComment = strpos($line, '#');
$rest = trim(substr($line, 0, -(strlen($line) - $posComment)));
if ($rest) {
$featureTokens = explode(',', $rest);
$features = [];
foreach ($featureTokens as $token) {
list($dimension, $weight) = explode('::', $token);
$features[$dimension] = (double) $weight;
}
// find record
/** @var Record $record */
$record = Record::findOne(['id' => $titles[$i]['id']]);
$i++;
$out = $testFile;
$label = 0;
$tasks = $record->getRecordTasks()->andWhere('is_prediction = 0')->all();
if ($tasks) {
$out = $trainFile;
$label = $tasks[0]->task_id;
} else {
$testIds[] = $record->id;
}
fwrite($out, $label . ' ');
$parts = ['1:' . $record->window_id, '2:' . $record->window->process_id];
foreach ($features as $dim => $weight) {
$parts[] = $dim . ':' . $weight;
}
fwrite($out, implode(' ', $parts));
fwrite($out, PHP_EOL);
}
}
fclose($trainFile);
fclose($testFile);
// train libsvm model
// best c=0.5, g=0.0078125
// best c=128.0, g=0.001953125, rate=67.3947
// best c=0.5, g=0.0078125, rate=99.5342
echo 'Run svm-train' . PHP_EOL;
$trainCommnad = '/home/alx/soft/libsvm-3.20/svm-train -b 1 -c "0.5" -g "0.0078125" titles.train.svm svm.model';
exec($trainCommnad);
// predict
echo 'Run svm-predict' . PHP_EOL;
$predictCommand = '/home/alx/soft/libsvm-3.20/svm-predict -b 1 -q titles.test.svm svm.model title.predict.txt';
exec($predictCommand);
$predictionFile = fopen('title.predict.txt', 'r');
$i = 0;
$transaction = \Yii::$app->db->beginTransaction();
// skip first line
$dummyLine = fgets($predictionFile);
$skiped = 0;
while (($line = fgets($predictionFile)) !== false) {
$line = trim($line);
if (!isset($testIds[$i])) {
throw new Exception('Что-то не сходится!');
}
$recordId = $testIds[$i];
$i++;
// Remove all predictions for this record
RecordTask::deleteAll(['record_id' => $recordId, 'is_prediction' => 1]);
$parts = explode(' ', $line);
$predictTask = (int) $parts[0];
$probability = array_slice($parts, 1);
$max = max(array_map('floatval', $probability));
if ($max < 0.75) {
// echo "skip (with {$max})".PHP_EOL;
$skiped++;
continue;
}
// Save new prediction
$model = new RecordTask();
$model->task_id = $predictTask;
$model->record_id = $recordId;
$model->is_prediction = 1;
$model->save(false);
}
$transaction->commit();
echo "Skiped {$skiped} records\n";
}
public function afterDelete()
{
RecordTask::deleteAll(['task_id' => $this->id]);
parent::afterDelete();
}
public function actionAssign()
{
$taskId = Yii::$app->request->getBodyParam('task');
$windowId = Yii::$app->request->getBodyParam('window');
$processId = Yii::$app->request->getBodyParam('process');
$transaction = Yii::$app->db->beginTransaction();
$timezone = new \DateTimeZone(Yii::$app->timeZone);
$from = new \DateTime('today', $timezone);
$to = new \DateTime('tomorrow', $timezone);
try {
$ids = Record::find()->with('window')->select(['id'])->andFilterWhere(['window_id' => $windowId, 'window.process_id' => $processId])->andWhere(['>=', 'start', $from->format('Y-m-d H:i:s')])->andWhere(['<', 'end', $to->format('Y-m-d H:i:s')])->createCommand()->queryColumn();
RecordTask::deleteAll(['record_id' => $ids]);
$values = array_map(function ($id) use($taskId) {
return [$id, $taskId];
}, $ids);
Yii::$app->db->createCommand()->batchInsert('{{record_task}}', ['record_id', 'task_id'], $values)->execute();
$transaction->commit();
} catch (Exception $e) {
$transaction->rollBack();
}
}