Content Scraping with VuGen
As I’ve said before, VuGen makes a great content scraping tool for cases when you want a quick and dirty script to save specific data from multiple webpages.
In this example, I wanted to create a list of all the WordPress plugins available from http://wordpress.org/extend/plugins/ (currently there are 4,245), and save all the metadata about the plugin:
- Number of downloads
- Version number
- Rating
- etc…
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | vuser_init() { // Only download content from "wordpress.org" web_add_auto_filter("Action=Include", "Host=wordpress.org", LAST); return 0; } /* Load the next "browse by popularity" page, then load each plugin page in turn. Save version and statistics data for each plugin to a file. There are 332 pages, so set number of iterations to 332 in Runtime Settings. http://wordpress.org/extend/plugins/browse/popular/page/2 - http://wordpress.org/extend/plugins/browse/popular/page/332 */ Action() { int i; char* file = "C:\\TEMP\\output.txt"; lr_start_transaction("load page"); web_reg_save_param("PluginName", "LB=\">", "RB=</a></h3>", "Ord=All", "Search=Body", LAST); web_reg_save_param("PluginURL", "LB=<h3><a href=\"", "RB=\">", "Ord=All", "Search=Body", LAST); web_reg_save_param("NumberOfDownloads", "LB=<li><span class=\"info-marker\">Downloads</span> ", "RB=</li>", "Ord=All", "Search=Body", LAST); web_reg_find("Text=WordPress › Most Popular « WordPress Plugins", LAST); web_reg_find("Text=<span class='page-numbers current'>{IterationNum}</span>", LAST); web_url("Popular Plugins", "URL=http://wordpress.org/extend/plugins/browse/popular/page/{IterationNum}", "TargetFrame=", "Resource=0", "RecContentType=text/html", "Referer=", "Snapshot=t1.inf", "Mode=HTML", LAST); lr_end_transaction("load page",LR_AUTO); // loop through all plugin pages that are linked from this page. for (i=1; i<=lr_paramarr_len("PluginName"); i++) { lr_start_transaction("load plugin page"); web_reg_save_param("Version", "LB=<strong>Version:</strong> ", "RB=</li>", "Ord=1", "Search=Body", LAST); web_reg_save_param("LastUpdated", "LB=<strong>Last Updated:</strong> ", "RB=<br />", "Ord=1", "Search=Body", LAST); web_reg_save_param("RequiresWordPressVersion", "LB=<strong>Requires WordPress Version:</strong> ", "RB=</li>", "Ord=1", "Search=Body", "NotFound=Warning", // this is an optional field LAST); web_reg_save_param("CompatibleUpTo", "LB=<strong>Compatible up to:</strong> ", "RB=</li>", "Ord=1", "Search=Body", "NotFound=Warning", // this is an optional field LAST); web_reg_save_param("AuthorHomepage", "LB=<li><a href='", "RB='>Author Homepage »</a>", "Ord=1", "Search=Body", "NotFound=Warning", // this is an optional field LAST); web_reg_save_param("PluginHomepage", "LB=<li><a href='", "RB='>Plugin Homepage »</a></li>", "Ord=1", "Search=Body", "NotFound=Warning", // this is an optional field LAST); web_reg_save_param("Rating", "LB=<div class=\"star star-rating\" style=\"width: ", "RB=px\"></div>", "Ord=1", "Search=Body", LAST); web_reg_save_param("NumberOfRatings", "LB=<span>(", "RB= ratings)</span>", "Ord=1", "Search=Body", LAST); web_reg_save_param("FileName", "LB=<p class=\"button\"><a href='http://downloads.wordpress.org/plugin/", "RB='>Download</a></p>", "Ord=1", "Search=Body", LAST); lr_save_string(lr_paramarr_idx("PluginURL", i), "URL"); lr_save_string(lr_paramarr_idx("PluginName", i), "Name"); lr_save_string(lr_paramarr_idx("NumberOfDownloads", i), "Downloads"); web_reg_find("Text=« WordPress Plugins", LAST); web_url("Plugin Page", "URL={URL}", "TargetFrame=", "Resource=0", "RecContentType=text/html", "Referer=http://wordpress.org/extend/plugins/browse/popular/", "Snapshot=t2.inf", "Mode=HTML", LAST); jds_append_to_file(file, lr_eval_string("{Name}\t" "{Version}\t" "{LastUpdated}\t" "{URL}\t" "{PluginHomepage}\t" "{AuthorHomepage}\t" "{RequiresWordPressVersion}\t" "{CompatibleUpTo}\t" "{Downloads}\t" "{Rating}\t" "{NumberOfRatings}\t" "{FileName}\t\n")); lr_end_transaction("load plugin page", LR_AUTO); } return 0; } |
For those who would like a copy of the raw data, it is available here (904 KB).
Related posts: