exclude external urls (catalyst#119)

Author: brendanheywood

README.md

@@ -6,6 +6,7 @@
 * [Installation](#installation)
 * [Configuration](#configuration)
 * [Testing](#testing)
+* [Debugging](#debugging)
 * [Reports](#reports)
 * [Support](#support)
 * [Warm thanks](#warm-thanks)
@@ -184,6 +185,14 @@ and
 
 /admin/tool/crawler/report.php?report=recent
 
+# Debugging
+
+You can also run link crawler on given page by passing url. You might need to Reset Progress if its still running from Administration > Reports > Link crawler -> Robot status
+
+```
+php admin/tool/crawler/cli/crawl-as.php --url=http://localhost/
+```
+
 # Reports
 
 4 new admin reports are available for showing the current crawl status, broken

classes/robot/crawler.php

@@ -333,9 +333,9 @@ public function mark_for_crawl($baseurl, $url, $courseid = null, $priority = TOO
         $isexcluded = false;
         // If this URL is external then check the ext whitelist.
         if (!self::is_external($url)) {
-            $excludes = str_replace("\r", '', self::get_config()->excludemdlurl);
+            $excludes = str_replace(PHP_EOL, ',', self::get_config()->excludemdlurl);
         } else {
-            $excludes = str_replace("\r", '', self::get_config()->excludeexturl);
+            $excludes = str_replace(PHP_EOL, ',', self::get_config()->excludemdlurl);
         }
 
         $isexcluded = profiling_string_matches($url, $excludes);

version.php

@@ -27,8 +27,8 @@
 defined('MOODLE_INTERNAL') || die();
 
 
-$plugin->version   = 2020030300;        // The current plugin version (Date: YYYYMMDDXX)
-$plugin->release   = 2020030300;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->version   = 2020030500;        // The current plugin version (Date: YYYYMMDDXX)
+$plugin->release   = 2020030500;        // The current plugin version (Date: YYYYMMDDXX)
 $plugin->requires  = 2016021800;        // Requires this Moodle version.
 $plugin->component = 'tool_crawler'; // To check on upgrade, that module sits in correct place.
 $plugin->maturity  = MATURITY_STABLE;