/** * The main feed fetching function. * Fetches the feed items from the source provided and inserts them into the DB. * * Called on hook 'wprss_fetch_single_feed_hook'. * * @since 3.2 */ function wprss_fetch_insert_single_feed_items($feed_ID) { // Check if the feed source is active. if (wprss_is_feed_source_active($feed_ID) === FALSE && wprss_feed_source_force_next_fetch($feed_ID) === FALSE) { // If it is not active ( paused ), return without fetching the feed items. return; } // If the feed source is forced for next fetch, remove the force next fetch data if (wprss_feed_source_force_next_fetch($feed_ID)) { delete_post_meta($feed_ID, 'wprss_force_next_fetch'); } // Get the feed source URL from post meta, and filter it $feed_url = get_post_meta($feed_ID, 'wprss_url', true); $feed_url = apply_filters('wprss_feed_source_url', $feed_url, $feed_ID); // Get the feed limit from post meta $feed_limit = get_post_meta($feed_ID, 'wprss_limit', true); // Sanitize the limit. If smaller or equal to zero, or an empty string, set to NULL. $feed_limit = $feed_limit <= 0 || empty($feed_limit) ? NULL : $feed_limit; // Filter the URL for validaty if (filter_var($feed_url, FILTER_VALIDATE_URL)) { // Get the feed items from the source $items = wprss_get_feed_items($feed_url, $feed_ID); // If got NULL, convert to an empty array if ($items === NULL) { $items = array(); } // If the feed has its own meta limit, if ($feed_limit !== NULL) { // slice the items array using the feed meta limit // @todo - Check current number of feed items for source, and delete oldest to make room for new, to keep to the limit. // Use wprss_get_feed_items_for_source $items_to_insert = array_slice($items, 0, $feed_limit); // Gather the permalinks of existing feed item's related to this feed source $existing_permalinks = get_existing_permalinks($feed_ID); // Generate a list of items fetched, that are not already in the DB $new_items = array(); foreach ($items as $item) { $permalink = $item->get_permalink(); if (!in_array($permalink, $existing_permalinks)) { if (!is_array($new_items)) { $new_items = array(); } $new_items = array_push($new_items, $item); } } // Get the number of feed items in DB, and their count $db_feed_items = wprss_get_feed_items_for_source($feed_ID); $num_db_feed_items = $db_feed_items->post_count; // Get the number of feed items we can store until we reach the limit $num_can_insert = $feed_limit - $num_db_feed_items; // Calculate how many feed items we must delete before importing, to keep to the limit $num_feed_items_to_delete = count($new_items) - $num_can_insert; // Get an array with the DB feed items in reverse order (oldest first) $db_feed_items_reversed = array_reverse($db_feed_items->posts); // Cut the array to get only the first few that are to be deleted ( equal to $num_feed_items_to_delete ) $feed_items_to_delete = array_slice($db_feed_items_reversed, 0, $num_feed_items_to_delete); // Iterate the feed items and delete them foreach ($feed_items_to_delete as $key => $post) { wp_delete_post($post->ID, TRUE); } //$items_to_insert = $new_items; } else { $items_to_insert = $items; } // Insert the items into the db if (!empty($items_to_insert)) { wprss_items_insert_post($items_to_insert, $feed_ID); } } }
/** * Show up the custom columns for the wprss_feed list * * @since 2.0 */ function wprss_show_custom_columns($column, $post_id) { switch ($column) { case 'errors': $errors = get_post_meta($post_id, 'wprss_error_last_import', true); $showClass = $errors === 'true' ? 'wprss-show' : ''; $msg = "This feed source experienced an error during the last feed fetch or validation check. Re-check the feed source URL or check the Error Log in the Debugging page for more details."; echo "<i title=\"{$msg}\" class=\"fa fa-warning fa-fw wprss-feed-error-symbol {$showClass}\"></i>"; break; case 'state': $active = wprss_is_feed_source_active($post_id); $text = $active ? 'Active' : 'Paused'; $button = $active ? 'Pause this feed source' : 'Activate this feed source'; $icon = $active ? 'pause' : 'play'; $value = $active ? 'paused' : 'active'; $indicator = $active ? 'green' : 'grey'; ?> <p> <span class="wprss-indicator-<?php echo $indicator; ?> " title="<?php echo $text; ?> "> <i class="fa fa-circle"></i> </span> <input type="hidden" name="wprss-redirect" value="1" /> <button type="submit" class='button-secondary' title="<?php echo $button; ?> " name="wprss-feed-id" value="<?php echo $post_id; ?> "> <i class='fa fa-<?php echo $icon; ?> '></i> </button> </p> <?php break; case 'updates': // Get the update interval $update_interval = get_post_meta($post_id, 'wprss_update_interval', TRUE); // Get the last updated and next update data $last_update = get_post_meta($post_id, 'wprss_last_update', TRUE); $last_update_items = get_post_meta($post_id, 'wprss_last_update_items', TRUE); $next_update = wprss_get_next_feed_source_update($post_id); // If using the global interval, get the timestamp of the next global update if ($update_interval === wprss_get_default_feed_source_update_interval() || $update_interval === '') { $next_update = wp_next_scheduled('wprss_fetch_all_feeds_hook', array()); } // Update the meta field if (wprss_is_feed_source_active($post_id)) { $next_update_text = $next_update === FALSE ? 'None' : human_time_diff($next_update, time()); } else { $next_update_text = 'Paused'; } update_post_meta($post_id, 'wprss_next_update', $next_update_text); ?> <p> Next update: <code class="next-update"> <?php echo $next_update_text; ?> </code> </p> <?php if ($last_update !== '') { ?> <p class="last-update-container"> Last updated: <code class="last-update"><?php echo human_time_diff($last_update, time()); ?> <?php _e('ago'); ?> </code> <?php if ($last_update_items !== '') { ?> <span class="last-update-imported-container"><br/>Last update imported: <code class="last-update-imported"><?php echo $last_update_items; ?> </code> items</span> <?php } ?> </p> <?php } break; case 'feed-count': $items = wprss_get_feed_items_for_source($post_id); $seconds_for_next_update = wprss_get_next_feed_source_update($post_id) - time(); $showClass = $seconds_for_next_update < 10 && $seconds_for_next_update > 0 || wprss_is_feed_source_deleting($post_id) ? 'wprss-show' : ''; echo '<p>'; echo "<span class=\"items-imported\">{$items->post_count}</span>"; echo "<i class=\"fa fa-fw fa-refresh fa-spin wprss-updating-feed-icon {$showClass}\" title=\"Updating feed source\"></i>"; echo '</p>'; // Set meta field for items imported update_post_meta($post_id, 'wprss_items_imported', $items->post_count); break; } }
/** * */ function wprss_feed_source_updates() { $response = array(); if (!current_user_can('edit_feed_sources')) { return $response; } if (empty($_POST['wprss_heartbeat'])) { return $response; } // Get the wprss heartbeat data and extract the data $wprss_heartbeat = $_POST['wprss_heartbeat']; extract($wprss_heartbeat); // Perform the action specified by the heartbeat data switch ($action) { /* FEED SOURCE UPDATING STATUS * Used to determine whether or not to show the updating icon in the feed source table. */ case 'feed_sources': // Prepare array of IDs for feed sources currently updating $feed_sources_data = array(); // Iterate all feed sources foreach ($params as $feed_id) { $feed_sources_data[$feed_id] = array(); $feed_source_data =& $feed_sources_data[$feed_id]; // Check if the feed source is updating $seconds_for_next_update = wprss_get_next_feed_source_update($feed_id) - time(); $feed_source_data['updating'] = $seconds_for_next_update < 2 && $seconds_for_next_update > 0 || wprss_is_feed_source_updating($feed_id) || wprss_is_feed_source_deleting($feed_id); // Add the number of imported items $items = wprss_get_feed_items_for_source($feed_id); $feed_source_data['items'] = $items->post_count; // Update the meta field update_post_meta($feed_id, 'wprss_items_imported', $items->post_count); // Add the next update time $next_update = wprss_get_next_feed_source_update($feed_id); $update_interval = get_post_meta($feed_id, 'wprss_update_interval', TRUE); // If using the global interval, get the timestamp of the next global update if ($update_interval === wprss_get_default_feed_source_update_interval() || $update_interval === '') { $next_update = wp_next_scheduled('wprss_fetch_all_feeds_hook', array()); } // Set the text appropriately if (!wprss_is_feed_source_active($feed_id)) { $feed_source_data['next-update'] = __('Paused', WPRSS_TEXT_DOMAIN); } elseif ($next_update === FALSE) { $feed_source_data['next-update'] = __('None', WPRSS_TEXT_DOMAIN); } else { $feed_source_data['next-update'] = human_time_diff($next_update, time()); } // Update the meta field update_post_meta($feed_id, 'wprss_next_update', $feed_source_data['next-update']); // Add the last update information $last_update = get_post_meta($feed_id, 'wprss_last_update', TRUE); $last_update_items = get_post_meta($feed_id, 'wprss_last_update_items', TRUE); $feed_source_data['last-update'] = $last_update === '' ? '' : human_time_diff($last_update, time()); $feed_source_data['last-update-imported'] = $last_update_items; // Add any error info $errors = get_post_meta($feed_id, 'wprss_error_last_import', true); $feed_source_data['errors'] = $errors; } // Send back all the IDs $response['wprss_feed_sources_data'] = $feed_sources_data; break; } // Return the response die(json_encode($response)); }
/** * The main feed fetching function. * Fetches the feed items from the source provided and inserts them into the DB. * * Called on hook 'wprss_fetch_single_feed_hook'. * * @since 3.2 */ function wprss_fetch_insert_single_feed_items($feed_ID) { // Check if the feed source is active. if (wprss_is_feed_source_active($feed_ID) === FALSE && wprss_feed_source_force_next_fetch($feed_ID) === FALSE) { // If it is not active ( paused ), return without fetching the feed items. return; } // If the feed source is forced for next fetch, remove the force next fetch data if (wprss_feed_source_force_next_fetch($feed_ID)) { delete_post_meta($feed_ID, 'wprss_force_next_fetch'); } update_post_meta($feed_ID, 'wprss_feed_is_updating', time()); // Get the feed source URL from post meta, and filter it $feed_url = get_post_meta($feed_ID, 'wprss_url', true); $feed_url = apply_filters('wprss_feed_source_url', $feed_url, $feed_ID); // Get the feed limit from post meta $feed_limit = get_post_meta($feed_ID, 'wprss_limit', true); // If the feed has no individual limit if ($feed_limit === '' || intval($feed_limit) <= 0) { // Get the global limit $global_limit = wprss_get_general_setting('limit_feed_items_imported'); // If no global limit is set, mark as NULL if ($global_limit === '' || intval($global_limit) <= 0) { $feed_limit = NULL; } else { $feed_limit = $global_limit; } } // Filter the URL for validaty if (filter_var($feed_url, FILTER_VALIDATE_URL)) { // Get the feed items from the source $items = wprss_get_feed_items($feed_url, $feed_ID); // If got NULL, convert to an empty array if ($items === NULL) { $items = array(); } // If using a limit ... if ($feed_limit === NULL) { $items_to_insert = $items; } else { $items_to_insert = array_slice($items, 0, $feed_limit); } // Gather the permalinks of existing feed item's related to this feed source $existing_permalinks = get_existing_permalinks($feed_ID); // Generate a list of items fetched, that are not already in the DB $new_items = array(); foreach ($items_to_insert as $item) { $permalink = wprss_normalize_permalink($item->get_permalink()); // Check if not blacklisted and not already imported if (wprss_is_blacklisted($permalink) === FALSE && !in_array(trim($permalink), $existing_permalinks)) { $new_items[] = $item; } } $items_to_insert = $new_items; // If using a limit - delete any excess items to make room for the new items if ($feed_limit !== NULL) { // Get the number of feed items in DB, and their count $db_feed_items = wprss_get_feed_items_for_source($feed_ID); $num_db_feed_items = $db_feed_items->post_count; // Get the number of feed items we can store until we reach the limit $num_can_insert = $feed_limit - $num_db_feed_items; // Calculate how many feed items we must delete before importing, to keep to the limit $num_feed_items_to_delete = count($new_items) - $num_can_insert; // Get an array with the DB feed items in reverse order (oldest first) $db_feed_items_reversed = array_reverse($db_feed_items->posts); // Cut the array to get only the first few that are to be deleted ( equal to $num_feed_items_to_delete ) $feed_items_to_delete = array_slice($db_feed_items_reversed, 0, $num_feed_items_to_delete); // Iterate the feed items and delete them foreach ($feed_items_to_delete as $key => $post) { wp_delete_post($post->ID, TRUE); } } update_post_meta($feed_ID, 'wprss_last_update', time()); // Insert the items into the db if (!empty($items_to_insert)) { wprss_items_insert_post($items_to_insert, $feed_ID); } } else { wprss_log("The feed URL is not valid! Please recheck."); } $next_scheduled = get_post_meta($feed_ID, 'wprss_reschedule_event', TRUE); if ($next_scheduled !== '') { wprss_feed_source_update_start_schedule($feed_ID); delete_post_meta($feed_ID, 'wprss_reschedule_event'); } delete_post_meta($feed_ID, 'wprss_feed_is_updating'); }
/** * Show up the custom columns for the wprss_feed list * * @since 2.0 */ function wprss_show_custom_columns($column, $post_id) { switch ($column) { case 'url': $url = get_post_meta($post_id, 'wprss_url', true); echo '<a href="' . esc_url($url) . '">' . esc_url($url) . '</a>'; break; case 'description': $description = get_post_meta($post_id, 'wprss_description', true); echo esc_html($description); break; case 'id': echo esc_html($post_id); break; case 'state': $active = wprss_is_feed_source_active($post_id); $text = $active ? 'Active' : 'Paused'; $button = $active ? 'Pause this feed source' : 'Activate this feed source'; $icon = $active ? 'pause' : 'play'; $value = $active ? 'paused' : 'active'; $indicator = $active ? 'green' : 'grey'; ?> <p> <span class="wprss-indicator-<?php echo $indicator; ?> " title="<?php echo $text; ?> "> <i class="fa fa-circle"></i> </span> <input type="hidden" name="wprss-redirect" value="1" /> <button type="submit" class='button-secondary' title="<?php echo $button; ?> " name="wprss-feed-id" value="<?php echo $post_id; ?> "> <i class='fa fa-<?php echo $icon; ?> '></i> </button> </p> <?php break; case 'next-update': $interval = get_post_meta($post_id, 'wprss_update_interval', TRUE); $timestamp = wprss_get_next_feed_source_update($post_id); // If using the global interval, get the timestamp of the next glboal update if ($interval === wprss_get_default_feed_source_update_interval() || $interval === '') { $timestamp = wp_next_scheduled('wprss_fetch_all_feeds_hook', array()); } ?> <p> <code> <?php if (!wprss_is_feed_source_active($post_id)) { ?> Paused <?php } elseif ($timestamp === FALSE) { ?> None <?php } else { ?> <?php echo human_time_diff($timestamp, time()); ?> <?php } ?> </code> </p> <?php break; case 'feed-count': $items = wprss_get_feed_items_for_source($post_id); echo '<p>' . $items->post_count . '</p>'; break; } }
/** * The main feed fetching function. * Fetches the feed items from the source provided and inserts them into the DB. * * Called on hook 'wprss_fetch_single_feed_hook'. * * @since 3.2 */ function wprss_fetch_insert_single_feed_items($feed_ID) { wprss_log_obj('Starting import of feed', $feed_ID, null, WPRSS_LOG_LEVEL_INFO); global $wprss_importing_feed; $wprss_importing_feed = $feed_ID; register_shutdown_function('wprss_detect_exec_timeout'); // Check if the feed source is active. if (wprss_is_feed_source_active($feed_ID) === FALSE && wprss_feed_source_force_next_fetch($feed_ID) === FALSE) { // If it is not active ( paused ), return without fetching the feed items. wprss_log('Feed is not active and not forced. Import cancelled.', null, WPRSS_LOG_LEVEL_INFO); return; } // If the feed source is forced for next fetch, remove the force next fetch data if (wprss_feed_source_force_next_fetch($feed_ID)) { delete_post_meta($feed_ID, 'wprss_force_next_fetch'); wprss_log('Force feed flag removed', null, WPRSS_LOG_LEVEL_SYSTEM); } $start_of_update = wprss_flag_feed_as_updating($feed_ID); wprss_log_obj('Start of import time updated', date('Y-m-d H:i:s', $start_of_update), null, WPRSS_LOG_LEVEL_SYSTEM); // Get the feed source URL from post meta, and filter it $feed_url = get_post_meta($feed_ID, 'wprss_url', true); wprss_log_obj('Original feed source URL', $feed_url, null, WPRSS_LOG_LEVEL_SYSTEM); $feed_url = apply_filters('wprss_feed_source_url', $feed_url, $feed_ID); wprss_log_obj('Actual feed source URL', $feed_url, null, WPRSS_LOG_LEVEL_INFO); // Get the feed limit from post meta $feed_limit = get_post_meta($feed_ID, 'wprss_limit', true); wprss_log_obj('Feed limit value is', $feed_limit, null, WPRSS_LOG_LEVEL_SYSTEM); // If the feed has no individual limit if ($feed_limit === '' || intval($feed_limit) <= 0) { wprss_log_obj('Using global limit', $feed_limit, null, WPRSS_LOG_LEVEL_NOTICE); // Get the global limit $global_limit = wprss_get_general_setting('limit_feed_items_imported'); // If no global limit is set, mark as NULL if ($global_limit === '' || intval($global_limit) <= 0) { $feed_limit = NULL; } else { $feed_limit = $global_limit; } } wprss_log_obj('Feed import limit', $feed_limit, null, WPRSS_LOG_LEVEL_INFO); // Filter the URL for validaty if (wprss_validate_url($feed_url)) { wprss_log_obj('Feed URL is valid', $feed_url, null, WPRSS_LOG_LEVEL_INFO); // Get the feed items from the source $items = wprss_get_feed_items($feed_url, $feed_ID); // If got NULL, convert to an empty array if ($items === NULL) { $items = array(); wprss_log('Items were NULL. Using empty array', null, WPRSS_LOG_LEVEL_WARNING); } // If using a limit ... if ($feed_limit === NULL) { $items_to_insert = $items; } else { $items_to_insert = array_slice($items, 0, $feed_limit); wprss_log_obj('Sliced a segment of items', count($items_to_insert), null, WPRSS_LOG_LEVEL_SYSTEM); } // Gather the permalinks of existing feed item's related to this feed source $existing_permalinks = wprss_get_existing_permalinks($feed_ID); wprss_log_obj('Retrieved existing permalinks', count($existing_permalinks), null, WPRSS_LOG_LEVEL_SYSTEM); // Check if we should only import uniquely-titled feed items. $existing_titles = array(); $unique_titles = FALSE; if (wprss_get_general_setting('unique_titles')) { $unique_titles = TRUE; $existing_titles = wprss_get_existing_titles(); wprss_log_obj('Retrieved existing titles from global', count($existing_titles), null, WPRSS_LOG_LEVEL_SYSTEM); } else { if (get_post_meta($feed_ID, 'wprss_unique_titles', true) === 'true') { $unique_titles = TRUE; $existing_titles = wprss_get_existing_titles($feed_ID); wprss_log_obj('Retrieved existing titles from feed source', count($existing_titles), null, WPRSS_LOG_LEVEL_SYSTEM); } } // Generate a list of items fetched, that are not already in the DB $new_items = array(); foreach ($items_to_insert as $item) { $permalink = wprss_normalize_permalink($item->get_permalink()); wprss_log_obj('Normalizing permalink', sprintf('%1$s -> %2$s', $item->get_permalink(), $permalink), null, WPRSS_LOG_LEVEL_SYSTEM); // Check if not blacklisted and not already imported $is_blacklisted = wprss_is_blacklisted($permalink); $permalink_exists = array_key_exists($permalink, $existing_permalinks); $title_exists = array_key_exists($item->get_title(), $existing_titles); if ($is_blacklisted === FALSE && $permalink_exists === FALSE && $title_exists === FALSE) { $new_items[] = $item; wprss_log_obj('Permalink OK', $permalink, null, WPRSS_LOG_LEVEL_SYSTEM); if ($unique_titles) { $existing_titles[$item->get_title()] = 1; } } else { if ($is_blacklisted) { wprss_log('Permalink blacklisted', null, WPRSS_LOG_LEVEL_SYSTEM); } if ($permalink_exists) { wprss_log('Permalink already exists', null, WPRSS_LOG_LEVEL_SYSTEM); } if ($title_exists) { wprss_log('Title already exists', null, WPRSS_LOG_LEVEL_SYSTEM); } } } $original_count = count($items_to_insert); $new_count = count($new_items); if ($new_count !== $original_count) { wprss_log_obj('Items filtered out', $original_count - $new_count, null, WPRSS_LOG_LEVEL_SYSTEM); } else { wprss_log('Items to import remained untouched. Not items already exist or are blacklisted.', null, WPRSS_LOG_LEVEL_SYSTEM); } $items_to_insert = $new_items; // If using a limit - delete any excess items to make room for the new items if ($feed_limit !== NULL) { wprss_log_obj('Some items may be deleted due to limit', $feed_limit, null, WPRSS_LOG_LEVEL_SYSTEM); // Get the number of feed items in DB, and their count $db_feed_items = wprss_get_feed_items_for_source($feed_ID); $num_db_feed_items = $db_feed_items->post_count; // Get the number of feed items we can store until we reach the limit $num_can_insert = $feed_limit - $num_db_feed_items; // Calculate how many feed items we must delete before importing, to keep to the limit $num_new_items = count($new_items); $num_feed_items_to_delete = $num_can_insert > $num_new_items ? 0 : $num_new_items - $num_can_insert; // Get an array with the DB feed items in reverse order (oldest first) $db_feed_items_reversed = array_reverse($db_feed_items->posts); // Cut the array to get only the first few that are to be deleted ( equal to $num_feed_items_to_delete ) $feed_items_to_delete = array_slice($db_feed_items_reversed, 0, $num_feed_items_to_delete); wprss_log(sprintf('There already are %1$d items in the database. %2$d items can be inserted. %3$d items will be deleted', $num_db_feed_items, $num_can_insert, $num_feed_items_to_delete), null, WPRSS_LOG_LEVEL_SYSTEM); // Iterate the feed items and delete them foreach ($feed_items_to_delete as $key => $post) { wp_delete_post($post->ID, TRUE); } if ($deleted_items_count = count($feed_items_to_delete)) { wprss_log_obj('Items deleted due to limit', $deleted_items_count, null, WPRSS_LOG_LEVEL_NOTICE); } } update_post_meta($feed_ID, 'wprss_last_update', $last_update_time = time()); update_post_meta($feed_ID, 'wprss_last_update_items', 0); wprss_log_obj('Last import time updated', $last_update_time, null, WPRSS_LOG_LEVEL_SYSTEM); // Insert the items into the db if (!empty($items_to_insert)) { wprss_log_obj('There are items to insert', count($items_to_insert), null, WPRSS_LOG_LEVEL_INFO); wprss_items_insert_post($items_to_insert, $feed_ID); } } else { wprss_log_obj('The feed URL is not valid! Please recheck', $feed_url); } $next_scheduled = get_post_meta($feed_ID, 'wprss_reschedule_event', TRUE); if ($next_scheduled !== '') { wprss_feed_source_update_start_schedule($feed_ID); delete_post_meta($feed_ID, 'wprss_reschedule_event'); wprss_log('Next update rescheduled', null, WPRSS_LOG_LEVEL_SYSTEM); } wprss_flag_feed_as_idle($feed_ID); wprss_log_obj('Import complete', $feed_ID, __FUNCTION__, WPRSS_LOG_LEVEL_INFO); }
/** * Deletes the required number of feed items for the given source, * to keep the number of feed items below its limit. * * @since 4.2 * @deprecated */ function wprss_truncate_feed_items_for_source($source) { // Get the limit setting $limit = get_post_meta($source, 'wprss_limit', true); // Calculate the number of feed items to delete $feed_items = wprss_get_feed_items_for_source($source); $n = intval($feed_items->found_posts) - intval($limit); // Delete the feed items wprss_delete_oldest_feed_items($n, $source); }