function perform_github_version_only_diff($project, $update, $previous_revision) { require_once 'include/memcache_functions.php'; global $CDASH_MEMCACHE_ENABLED, $CDASH_MEMCACHE_PREFIX, $CDASH_MEMCACHE_SERVER; $current_revision = $update->Revision; // Check if we have a Github account associated with this project. // If so, we are much less likely to get rate-limited by the API. $auth = array(); $repositories = $project->GetRepositories(); foreach ($repositories as $repo) { if (strlen($repo['username']) > 0 && strlen($repo['password']) > 0) { $auth = ['auth' => [$repo['username'], $repo['password']]]; break; } } // Connect to memcache. if ($CDASH_MEMCACHE_ENABLED) { list($server, $port) = $CDASH_MEMCACHE_SERVER; $memcache = cdash_memcache_connect($server, $port); // Disable memcache for this request if it fails to connect. if ($memcache === false) { $CDASH_MEMCACHE_ENABLED = false; } } // Check if we've memcached the difference between these two revisions. $diff_response = null; $diff_key = "{$CDASH_MEMCACHE_PREFIX}:{$project->Name}:{$current_revision}:{$previous_revision}"; if ($CDASH_MEMCACHE_ENABLED) { $cached_response = cdash_memcache_get($memcache, $diff_key); if ($cached_response !== false) { $diff_response = $cached_response; } } if (is_null($diff_response)) { // Use the GitHub API to find what changed between these two revisions. // This API endpoint takes the following form: // GET /repos/:owner/:repo/compare/:base...:head $base_api_url = get_github_api_url($project->CvsUrl); $client = new GuzzleHttp\Client(); $api_url = "{$base_api_url}/compare/{$previous_revision}...{$current_revision}"; try { $response = $client->request('GET', $api_url, $auth); } catch (GuzzleHttp\Exception\ClientException $e) { // Typically this occurs due to a local commit that GitHub does not // know about. add_log($e->getMessage(), "perform_github_version_only_diff", LOG_WARNING, $project->Id); return; } $diff_response = strval($response->getBody()); // Cache the response from the GitHub API for 24 hours. if ($CDASH_MEMCACHE_ENABLED) { cdash_memcache_set($memcache, $diff_key, $diff_response, 60 * 60 * 24); } } $response_array = json_decode($diff_response, true); // To do anything meaningful here our response needs to tell us about commits // and the files that changed. Abort early if either of these pieces of // information are missing. if (!is_array($response_array) || !array_key_exists('commits', $response_array) || !array_key_exists('files', $response_array)) { return; } // Discard merge commits. We want to assign credit to the author who did // the actual work, not the approver who clicked the merge button. foreach ($response_array['commits'] as $idx => $commit) { if (strpos($commit['commit']['message'], 'Merge pull request') !== false) { unset($response_array['commits'][$idx]); } } // If we still have more than one commit, we'll need to perform follow-up // API calls to figure out which commit was likely responsible for each // changed file. $multiple_commits = false; if (count($response_array['commits']) > 1) { $multiple_commits = true; // Generate list of commits contained by this changeset in reverse order // (most recent first). $list_of_commits = array_reverse($response_array['commits']); // Also maintain a local cache of what files were changed by each commit. // This prevents us from hitting the GitHub API more than necessary. $cached_commits = array(); } $pdo = get_link_identifier()->getPdo(); // Find the commit that changed each file. foreach ($response_array['files'] as $modified_file) { if ($multiple_commits) { // Find the most recent commit that changed this file. $commit = null; // First check our local cache. foreach ($cached_commits as $sha => $files) { if (in_array($modified_file['filename'], $files)) { $idx = array_search($sha, array_column($list_of_commits, 'sha')); $commit = $list_of_commits[$idx]; break; } } if (is_null($commit)) { // Next, check the database. $stmt = $pdo->prepare('SELECT DISTINCT revision FROM updatefile WHERE filename=?'); $stmt->execute(array($modified_file['filename'])); while ($row = $stmt->fetch()) { foreach ($list_of_commits as $c) { if ($row['revision'] == $c['sha']) { $commit = $c; break; } } if (!is_null($commit)) { break; } } } if (is_null($commit)) { // Lastly, use the Github API to find what files this commit changed. // To avoid being rate-limited, we only perform this lookup once // per commit, caching the results as we go. foreach ($list_of_commits as $c) { $sha = $c['sha']; if (array_key_exists($sha, $cached_commits)) { // We already looked up this commit. // Apparently it didn't modify the file we're looking for. continue; } $commit_response = null; $commit_key = "{$CDASH_MEMCACHE_PREFIX}:{$project->Name}:{$sha}"; if ($CDASH_MEMCACHE_ENABLED) { // Check memcache if it is enabled before hitting // the GitHub API. $cached_response = cdash_memcache_get($memcache, $commit_key); if ($cached_response !== false) { $commit_response = $cached_response; } } if (is_null($commit_response)) { $api_url = "{$base_api_url}/commits/{$sha}"; try { $r = $client->request('GET', $api_url, $auth); } catch (GuzzleHttp\Exception\ClientException $e) { add_log($e->getMessage(), "perform_github_version_only_diff", LOG_ERROR, $project->Id); break; } $commit_response = strval($r->getBody()); if ($CDASH_MEMCACHE_ENABLED) { // Cache this response for 24 hours. cdash_memcache_set($memcache, $commit_key, $commit_response, 60 * 60 * 24); } } $commit_array = json_decode($commit_response, true); if (!is_array($commit_array) || !array_key_exists('files', $commit_array)) { // Skip to the next commit if no list of files was returned. $cached_commits[$sha] = array(); continue; } // Locally cache what files this commit changed. $cached_commits[$sha] = array_column($commit_array['files'], 'filename'); // Check if this commit modified the file in question. foreach ($commit_array['files'] as $file) { if ($file['filename'] === $modified_file['filename']) { $commit = $c; break; } } if (!is_null($commit)) { // Stop examining commits once we find one that matches. break; } } } if (is_null($commit)) { // Skip this file if we couldn't find a commit that modified it. continue; } } else { $commit = $response_array['commits'][0]; } // Record this modified file as part of the changeset. $updateFile = new BuildUpdateFile(); $updateFile->Filename = $modified_file['filename']; $updateFile->CheckinDate = $commit['commit']['author']['date']; $updateFile->Author = $commit['commit']['author']['name']; $updateFile->Email = $commit['commit']['author']['email']; $updateFile->Committer = $commit['commit']['committer']['name']; $updateFile->CommitterEmail = $commit['commit']['committer']['email']; $updateFile->Log = $commit['commit']['message']; $updateFile->Revision = $commit['sha']; $updateFile->PriorRevision = $previous_revision; $updateFile->Status = 'MODIFIED'; $update->AddFile($updateFile); } $update->Insert(); return true; }
$has_subprojects = $Project->GetNumberOfSubProjects() > 0; // Handle optional date argument. @($date = $_GET['date']); if ($date != null) { $date = htmlspecialchars(pdo_real_escape_string($date)); } else { $date = date(FMT_DATE); } list($previousdate, $currentstarttime, $nextdate) = get_dates($date, $Project->NightlyTime); // Date range is currently hardcoded to two weeks in the past. // This could become a configurable value instead. $date_range = 14; // Use cache if it's enabled, use_cache isn't set to 0, and an entry exists in Memcache // Using cache is implied, but the user can set use_cache to 0 to explicitly disable it // (This is a good method of ensuring the cache for this page stays up) if ($CDASH_MEMCACHE_ENABLED && !(isset($_GET['use_cache']) && $_GET['use_cache'] == 0) && ($cachedResponse = cdash_memcache_get($memcache, cdash_memcache_key('overview'))) !== false) { echo $cachedResponse; return; } // begin JSON response that is used to render this page $response = begin_JSON_response(); get_dashboard_JSON_by_name($projectname, $date, $response); $response['title'] = "CDash Overview : {$projectname}"; $response['showcalendar'] = 1; $menu['previous'] = "overview.php?project={$projectname}&date={$previousdate}"; $menu['current'] = "overview.php?project={$projectname}"; $menu['next'] = "overview.php?project={$projectname}&date={$nextdate}"; $response['menu'] = $menu; $response['hasSubProjects'] = $has_subprojects; // configure/build/test data that we care about. $build_measurements = array('configure warnings', 'configure errors', 'build warnings', 'build errors', 'failing tests');