MasterPuppeteer

Updates broken fetches.

...@@ -236,12 +236,12 @@ public class CometTVPuppet implements InstallablePuppet { ...@@ -236,12 +236,12 @@ public class CometTVPuppet implements InstallablePuppet {
236 236
237 @Override 237 @Override
238 boolean isUnavailableIn(String region) { 238 boolean isUnavailableIn(String region) {
239 - return false 239 + return region != 'us'
240 } 240 }
241 241
242 @Override 242 @Override
243 String getPreferredRegion() { 243 String getPreferredRegion() {
244 - return null 244 + return 'us'
245 } 245 }
246 246
247 @Override 247 @Override
......
...@@ -77,7 +77,18 @@ public class PBSKidsPuppet implements InstallablePuppet { ...@@ -77,7 +77,18 @@ public class PBSKidsPuppet implements InstallablePuppet {
77 description += " (Ages " + item.getString("age_range") + ")" 77 description += " (Ages " + item.getString("age_range") + ")"
78 } 78 }
79 String url = "http://pbskids.org/pbsk/video/api/getVideos/?startindex=1&endindex=200&category=&group=&selectedID=&status=available&player=flash&flash=true&program=" + name.replace(" ","%20").replace("&", "%26") 79 String url = "http://pbskids.org/pbsk/video/api/getVideos/?startindex=1&endindex=200&category=&group=&selectedID=&status=available&player=flash&flash=true&program=" + name.replace(" ","%20").replace("&", "%26")
80 - String image = item.getJSONObject("images").getJSONObject("program-kids-square").getString("url") 80 + String image = null
81 + try {
82 + image = item.getJSONObject("images").getJSONObject("program-kids-square").getString("url")
83 + } catch (Exception ex) {
84 + try {
85 + image = item.getJSONObject("images").getJSONObject("originalres_4x3").getString("url")
86 + } catch (Exception exx) {
87 + try {
88 + image = item.getJSONObject("images").getJSONObject("kids-mezzannine-4x3").getString("url")
89 + } catch (ignore) { }
90 + }
91 + }
81 children.add(new PBSKidsPuppet(this, url, name, description, false, image)) 92 children.add(new PBSKidsPuppet(this, url, name, description, false, image))
82 } 93 }
83 } else if (mUrl.contains("/api/getVideos") && !mUrl.contains("&type=")) { 94 } else if (mUrl.contains("/api/getVideos") && !mUrl.contains("&type=")) {
......
...@@ -46,11 +46,11 @@ public class PBSThinkTVPuppet implements InstallablePuppet { ...@@ -46,11 +46,11 @@ public class PBSThinkTVPuppet implements InstallablePuppet {
46 public PBSThinkTVPuppet() { 46 public PBSThinkTVPuppet() {
47 this( 47 this(
48 null, 48 null,
49 - "http://www.pbs.org/shows-page/0/?genre=&title=&callsign=", 49 + "http://www.pbs.org/shows-page/0/?genre=&title=&callsign=&alphabetically=false",
50 "PBS Think TV", 50 "PBS Think TV",
51 "Your favorite PBS shows.", 51 "Your favorite PBS shows.",
52 - "https://superrepo.org/static/images/icons/original/xplugin.video.thinktv.png.pagespeed.ic.SmysxYmAM7.jpg", 52 + "https://images-na.ssl-images-amazon.com/images/I/51GpY5aDb8L.png",
53 - "https://superrepo.org/static/images/fanart/original/plugin.video.thinktv.jpg" 53 + "http://bento.cdn.pbs.org/hostedbento-prod/blog/20151102_211520_484137_pbs_bemore_bub_blu.png"
54 ) 54 )
55 } 55 }
56 56
...@@ -67,7 +67,9 @@ public class PBSThinkTVPuppet implements InstallablePuppet { ...@@ -67,7 +67,9 @@ public class PBSThinkTVPuppet implements InstallablePuppet {
67 PuppetIterator getChildren() { 67 PuppetIterator getChildren() {
68 PuppetIterator children = new PBSThinkTVPuppetIterator() 68 PuppetIterator children = new PBSThinkTVPuppetIterator()
69 69
70 - JSONArray json = new JSONObject(new URL(mUrl).getText()).getJSONObject("results").getJSONArray("content") 70 + JSONArray json = new JSONObject(new URL(mUrl).getText(requestProperties: [
71 + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
72 + ])).getJSONObject("results").getJSONArray("content")
71 73
72 for (int i = 0; i < json.length(); i++) { 74 for (int i = 0; i < json.length(); i++) {
73 JSONObject item = json.getJSONObject(i) 75 JSONObject item = json.getJSONObject(i)
...@@ -214,9 +216,9 @@ public class PBSThinkTVPuppet implements InstallablePuppet { ...@@ -214,9 +216,9 @@ public class PBSThinkTVPuppet implements InstallablePuppet {
214 216
215 public PBSThinkTVCategoryPuppet(ParentPuppet parent, JSONObject item) { 217 public PBSThinkTVCategoryPuppet(ParentPuppet parent, JSONObject item) {
216 mParent = parent 218 mParent = parent
217 - mId = item.getString("id") 219 + mId = item.getString("url")
218 mName = item.getString("title") 220 mName = item.getString("title")
219 - mDescription = item.getString("description") 221 + mDescription = item.has("description") && !item.isNull("description") ? item.getString("description") : null
220 mImageUrl = item.isNull("image") ? parent.getImageUrl() : item.getString("image") 222 mImageUrl = item.isNull("image") ? parent.getImageUrl() : item.getString("image")
221 } 223 }
222 224
...@@ -233,27 +235,28 @@ public class PBSThinkTVPuppet implements InstallablePuppet { ...@@ -233,27 +235,28 @@ public class PBSThinkTVPuppet implements InstallablePuppet {
233 PuppetIterator getChildren() { 235 PuppetIterator getChildren() {
234 PuppetIterator children = new PBSThinkTVPuppetIterator() 236 PuppetIterator children = new PBSThinkTVPuppetIterator()
235 if (mPage == 0) { 237 if (mPage == 0) {
236 - ["episodes", "previews", "clips"].each { videoType -> 238 + ["episodes", "extras"].each { videoType ->
237 children.add(new PBSThinkTVCategoryPuppet(this, mId, videoType.capitalize(), mImageUrl, videoType, mPage + 1)) 239 children.add(new PBSThinkTVCategoryPuppet(this, mId, videoType.capitalize(), mImageUrl, videoType, mPage + 1))
238 } 240 }
239 } else { 241 } else {
240 - String url = "http://www.pbs.org/show/" + mId + "/$mVideoType/?page=$mPage" 242 + String url = "http://www.pbs.org" + mId + mVideoType
241 - if (!Jsoup.connect(url).followRedirects(false).execute().hasHeader("location")) { 243 + Document document = Jsoup.connect(url)
242 - Document document = Jsoup.connect(url).ignoreContentType(true).get() 244 + .userAgent("Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6")
243 - int numItems = 0 245 + .ignoreContentType(true)
244 - document.select(".video-summary").each { node -> 246 + .get()
245 - String id = node.select("a").first().attr("href").replace("/video/", "").replace("/", "") 247 + int numItems = 0
246 - String name = node.select(".video-summary__title").first().text() 248 + document.select(".video-summary").each { node ->
247 - String duration = node.select(".video-summary__duration").first().text() 249 + String id = node.select(".btn--watchlist").attr("data-id")
248 - def images = node.select(".video-summary__image").attr("data-srcset").split() 250 + String name = node.select(".video-summary__title").first().text()
249 - String imageUrl = images[0] 251 + String duration = ""
250 - String backgroundImageUrl = images[images.length - 2] 252 + def images = node.select(".video-summary__image").attr("data-srcset").split()
251 - children.add(new PBSThinkTVSourcesPuppet(this, id, name, duration, imageUrl, backgroundImageUrl)) 253 + String imageUrl = images[0]
252 - numItems++ 254 + String backgroundImageUrl = images[images.length - 2]
253 - } 255 + children.add(new PBSThinkTVSourcesPuppet(this, id, name, duration, imageUrl, backgroundImageUrl))
254 - if (numItems > 23) { // 24 items per page so if we hit 24, present a next link 256 + numItems++
255 - children.add(new PBSThinkTVCategoryPuppet(this, mId, mVideoType.capitalize() + " page " + (mPage + 1), mImageUrl, mVideoType, mPage + 1)) 257 + }
256 - } 258 + if (numItems > 23) { // 24 items per page so if we hit 24, present a next link
259 + children.add(new PBSThinkTVCategoryPuppet(this, mId, mVideoType.capitalize() + " page " + (mPage + 1), mImageUrl, mVideoType, mPage + 1))
257 } 260 }
258 } 261 }
259 return children 262 return children
......