001package Torello.HTML.Tools.Images; 002 003import Torello.HTML.*; 004import Torello.Java.*; 005 006import Torello.Java.Additional.Ret2; 007 008import java.util.function.*; 009 010import java.io.Serializable; 011import java.io.File; 012import java.net.URL; 013import java.net.MalformedURLException; 014import java.util.Vector; 015import java.util.Objects; 016import java.util.concurrent.TimeUnit; 017import java.util.regex.Matcher; 018 019 020/** 021 * Holds all relevant configurations and parameters needed to run the primary download-loop of 022 * class {@link ImageScraper} 023 * 024 * <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST> 025 * <EMBED CLASS='external-html' DATA-FILE-ID=REQ_STR_BUILDER1_EX> 026 */ 027@SuppressWarnings("overrides") 028@Torello.JavaDoc.JDHeaderBackgroundImg(EmbedTagFileID="IMAGE_SCRAPER_CLASS") 029public class Request implements Cloneable, Serializable 030{ 031 /** <EMBED CLASS='external-html' DATA-FILE-ID=SVUID> */ 032 public static final long serialVersionUID = 1; 033 034 035 // ******************************************************************************************** 036 // ******************************************************************************************** 037 // MAIN CONSTRUCTORS 038 // ******************************************************************************************** 039 // ******************************************************************************************** 040 041 042 // There are 5 or 6 'static' builder-methods below. The only reason on earth that those are 043 // static-methods rather than constructors is that their parameter lists all use the same 044 // 'Iterable', but with a different Generic-Parameter. If you convert those to Constructors, 045 // you will get that they have the "Same Erasure", and that compiling cannot continue. 046 // 047 // Instead they are methods that have slightly different names, and the Java-Compiler, instead, 048 // shuts up, and stops complaining. 049 050 private Request( 051 Vector<URL> source, int size, URL originalPageURL, Vector<String[]> b64Images, 052 Vector<Exception> tagNodeSRCExceptions 053 ) 054 { 055 this.source = source; 056 this.size = size; 057 this.counterPrinter = getPrinter(size); 058 this.originalPageURL = originalPageURL; 059 this.b64Images = b64Images; 060 this.tagNodeSRCExceptions = tagNodeSRCExceptions; 061 } 062 063 private Request(Vector<URL> source, int size, URL originalPageURL) 064 { 065 this.source = source; 066 this.size = size; 067 this.counterPrinter = getPrinter(size); 068 this.originalPageURL = originalPageURL; 069 this.b64Images = null; 070 this.tagNodeSRCExceptions = null; 071 } 072 073 074 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 075 // Small static constructor-helper 076 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 077 078 private static IntFunction<String> getPrinter(int size) 079 { 080 // Now produce the Printer for the Image-Number. All this does is make sure to do an 081 // appropriate zero-padding for the text-output. 082 083 if (size < 10) return (int i) -> "" + i; 084 else if (size < 100) return StringParse::zeroPad10e2; 085 else if (size < 1000) return StringParse::zeroPad; 086 else if (size < 10000) return StringParse::zeroPad10e4; 087 088 // This case seems extremely unlikely and even largely preposterous, but leaving it like 089 // this means I will never have to analyze this crap ever again. Note that the above case 090 // where size is greater than 1,000 seems a little ridiculous. Usually there are under 100 091 // photos on any one HTML Page. 092 093 else 094 { 095 final int power = (int) Math.floor(Math.log10(size)); 096 return (int i) -> StringParse.zeroPad(i, power); 097 } 098 } 099 100 101 // ******************************************************************************************** 102 // ******************************************************************************************** 103 // Static Constructor-Like Builder Methods (Cannot Use Constructors because of "Erasure") 104 // ******************************************************************************************** 105 // ******************************************************************************************** 106 107 108 /** 109 * Builds an instance of this class from a list of {@code URL's} as {@code String's} 110 * 111 * @param source <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ITER_STR> 112 * 113 * @return A {@code 'Request'} instance. This may be further configured by assigning values to 114 * any / all fields (which will still have their initialized / default-values) 115 * 116 * @throws NullPointerException If any of the {@code String's} in the {@code Iterable} are null 117 * 118 * @throws IllegalArgumentException If any of the {@code URL's} are {@code String's} which 119 * begin with neither {@code 'http://'} nor {@code 'https://'}. Since this method doesn't 120 * accept the parameter {@code 'originalPageURL'}, each and every {@code URL} in the 121 * {@code 'source'} iterable must be a full & complete {@code URL}. 122 * 123 * <BR /><BR />This exception will also throw if there are any {@code URL's} in the 124 * {@code String}-List that cause a {@code MalformedURLException} to throw when constructing an 125 * instance of {@code java.net.URL} from the {@code String}. In these cases, the original 126 * {@code MalformedURLException} will be assigned to the {@code 'cause'}, and may be retrieved 127 * using the exception's {@code getCause()} method. 128 */ 129 public static Request buildFromStrIter(Iterable<String> source) 130 { 131 Vector<URL> temp = new Vector<>(); 132 int count = 0; 133 134 for (String urlStr : source) 135 { 136 count++; 137 138 if (urlStr == null) throw new NullPointerException( 139 "The " + count + StringParse.ordinalIndicator(count) + "th element of " + 140 "Iterable-Parameter 'source' is null" 141 ); 142 143 if (StrCmpr.startsWithNAND(urlStr, "http://", "https://")) 144 145 throw new IllegalArgumentException( 146 "The " + count + StringParse.ordinalIndicator(count) + "th element of " + 147 "Iterable-Parameter 'source' did neither begin with 'http://' nor " + 148 "'https://'. If there are any partial URL's in you Iterable, you must " + 149 "re-build using an 'originalPageURL' parameter in order to resolve any and " + 150 "all partial URL's." 151 ); 152 153 try 154 { temp.add(new URL(urlStr)); } 155 156 catch (MalformedURLException e) 157 { 158 throw new IllegalArgumentException( 159 "When attempting to build the " + count + StringParse.ordinalIndicator(count) + 160 " element of Iterable-Parameter 'source', a MalformedURLException threw. " + 161 "Please see e.getCause() for details.", 162 e 163 ); 164 } 165 } 166 167 return new Request(temp, count, null); 168 } 169 170 /** 171 * Builds an instance of this class from a list of {@code URL's} as {@code String's} 172 * 173 * @param source <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ITER_STR> 174 * @param originalPageURL <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ORIG_PG_URL> 175 * 176 * @param skipDontThrowIfBadStr If an exception is thrown when attempting to resolve a 177 * partial-{@code URL}, and this parameter is {@code TRUE}, then that exception is suppressed 178 * and logged, and the builder-loop continues to the next {@code URL}-as-a-{@code String}. 179 * 180 * <BR /><BR />When this parameter is passed {@code FALSE}, unresolvable {@code URL's} will 181 * generate an {@code IllegalArgumentException}-throw. 182 * 183 * <BR /><BR />Note that the presence of a null in the {@code Iterable 'source'} parameter 184 * will always force this method to throw {@code NullPointerException}. 185 * 186 * @return A {@code 'Request'} instance. This may be further configured by assigning values to 187 * any / all fields (which will still have their initialized / default-values) 188 * 189 * @throws NullPointerException If any of the {@code String's} in the {@code Iterable} are null 190 * 191 * @throws IllegalArgumentException This exception will also throw if there are any 192 * {@code URL's} in the {@code String}-List that cause a {@code MalformedURLException} to throw 193 * when constructing an instance of {@code java.net.URL} from the {@code String}. In these 194 * cases, the generated {@code MalformedURLException} will be assigned to the exception's 195 * {@code 'cause'}, and may therefore be retrieved using this exception's {@code getCause()} 196 * method. 197 */ 198 public static Request buildFromStrIter 199 (Iterable<String> source, URL originalPageURL, boolean skipDontThrowIfBadStr) 200 { 201 if (originalPageURL == null) throw new NullPointerException("'originalPageURL' is null."); 202 203 Vector<URL> temp = new Vector<>(); 204 Vector<Exception> strExceptions = new Vector<>(); 205 Vector<String[]> b64Dummy = new Vector<>(); 206 int count = 0; 207 208 for (String urlStr : source) 209 { 210 count++; 211 212 // This isn't used here, but the (private) Request-Constructor needs an empty Vector if 213 // there aren't any b64-Images. 214 215 b64Dummy.add(null); 216 217 if (urlStr == null) throw new NullPointerException( 218 "The " + count + StringParse.ordinalIndicator(count) + " element of " + 219 "Iterable-Parameter 'source' is null" 220 ); 221 222 Ret2<URL, MalformedURLException> r2 = Links.resolve_KE(urlStr, originalPageURL); 223 224 if (r2.b == null) temp.add(r2.a); 225 226 else 227 { 228 if (skipDontThrowIfBadStr) 229 { 230 temp.add(null); 231 strExceptions.add(r2.b); 232 } 233 234 else throw new IllegalArgumentException( 235 "When attempting to resolve the " + count + 236 StringParse.ordinalIndicator(count) + " TagNode of Iterable-Parameter " + 237 "'source' an Exception was thrown. See 'getCause' for details.", 238 r2.b 239 ); 240 } 241 242 } 243 244 return new Request(temp, count, originalPageURL, b64Dummy, strExceptions); 245 } 246 247 /** 248 * Builds an instance of this class using the {@code SRC}-Attribute from a list of 249 * {@link TagNode}'s. 250 * 251 * @param source <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ITER_TGND> 252 * 253 * @param originalPageURL <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ORIG_PG_URL> 254 * 255 * @param skipDontThrowIfBadSRCAttr 256 * <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_SKIP_BOOL> 257 * 258 * @return A {@code 'Request'} instance. This may be further configured by assigning values to 259 * any / all fields (which will still have their initialized / default-values) 260 * 261 * @throws NullPointerException If any of the {@link TagNode}'s in the {@code Iterable} are 262 * null 263 * 264 * @throws SRCException If any of the {@link TagNode}'s in the list do not have a {@code 'SRC'} 265 * Attribute, and {@code 'skipDontThrowIfBadSRCAttr'} is {@code FALSE}. 266 * 267 * <BR /><BR />This exception will also throw if there are any {@code URL's} in the 268 * {@link TagNode}-List that cause a {@code MalformedURLException} to throw when constructing 269 * an instance of {@code java.net.URL} (from the {@code TagNode's SRC}-Attribute). In these 270 * cases, the generated {@code MalformedURLException} will be assigned to the exception's 271 * {@code 'cause'}, and may therefore be retrieved using the exception's {@code getCause()} 272 * method. 273 * 274 * <BR /><BR />If {@code 'skipDontThrowIfBadSRCAttr'} is {@code FALSE}, then this 275 * exception will not throw, and a null will be placed in the query-list. 276 */ 277 public static Request buildFromTagNodeIter 278 (Iterable<TagNode> source, URL originalPageURL, boolean skipDontThrowIfBadSRCAttr) 279 { 280 if (originalPageURL == null) throw new NullPointerException("'originalPageURL' is null."); 281 282 Vector<URL> temp = new Vector<>(); 283 int count = 0; 284 Vector<String[]> b64Images = new Vector<>(); 285 Vector<Exception> tagNodeSRCExceptions = new Vector<>(); 286 287 for (TagNode tn : source) 288 { 289 count++; 290 291 if (tn == null) throw new NullPointerException( 292 "The " + count + StringParse.ordinalIndicator(count) + " element of " + 293 "Iterable-Parameter 'source' is null" 294 ); 295 296 String urlStr = tn.AV("src"); 297 298 if (urlStr == null) 299 { 300 SRCException e = new SRCException( 301 "The " + count + StringParse.ordinalIndicator(count) + " TagNode of " + 302 "Iterable-Parameter 'source' does not have a SRC-Attribute" 303 ); 304 305 if (skipDontThrowIfBadSRCAttr) 306 { 307 temp.add(null); 308 b64Images.add(null); 309 tagNodeSRCExceptions.add(e); 310 continue; 311 } 312 313 else throw e; 314 } 315 316 317 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 318 // Handle the Base-64 Stuff, here. 08.31.2023 319 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 320 321 Matcher m = IF.B64_INIT_STRING.matcher(urlStr); 322 323 if (m.find()) 324 { 325 temp.add(null); 326 327 b64Images.add(new String[] { m.group(1), m.group(2)}); 328 329 continue; 330 } 331 332 333 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 334 // Handle a normal URL. 08.31.2023 335 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 336 337 Ret2<URL, MalformedURLException> r2 = Links.resolve_KE(urlStr, originalPageURL); 338 339 if (r2.b != null) 340 { 341 SRCException e = new SRCException( 342 "Attempting to resolve the " + count + StringParse.ordinalIndicator(count) + 343 " TagNode of Iterable-Parameter 'source' there was an Exception. See " + 344 "'getCause' for details.", 345 r2.b 346 ); 347 348 if (skipDontThrowIfBadSRCAttr) 349 { 350 temp.add(null); 351 b64Images.add(null); 352 tagNodeSRCExceptions.add(e); 353 continue; 354 } 355 356 else throw e; 357 } 358 359 temp.add(r2.a); 360 } 361 362 return new Request(temp, count, originalPageURL, b64Images, tagNodeSRCExceptions); 363 } 364 365 /** 366 * Builds an instance of this class using the {@code SRC}-Attribute from a list of 367 * {@link TagNode}'s. 368 * 369 * @param source <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ITER_TGND> 370 * 371 * @param skipDontThrowIfBadSRCAttr 372 * <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_SKIP_BOOL> 373 * 374 * @return A {@code 'Request'} instance. This may be further configured by assigning values to 375 * any / all fields (which will still have their initialized / default-values) 376 * 377 * @throws NullPointerException If any of the {@link TagNode}'s in the {@code Iterable} are 378 * null 379 * 380 * @throws SRCException If any of the {@link TagNode}'s in the list do not have a 381 * {@code 'SRC'}-Attribute, and {@code 'skipDontThrowIfBadSRCAttr'} is {@code FALSE}. 382 * 383 * <BR /><BR />This exception will also throw if any of the {@code URL's} assigned to a 384 * {@code 'SRC'}-Attribute are partial-{@code URL's} which do not begin with {@code 'http://'} 385 * (or {@code 'https://'}), and {@code 'skipDontThrowIfBadSRCAttr'} is {@code FALSE}. 386 * 387 * <BR /><BR />Finally, if any of the {@code URL's} inside a {@link TagNode}'s' 388 * {@code 'SRC'}-Attribute cause a {@code MalformedURLException}, that exception will be 389 * assigned to the {@code cause} of a {@link SRCException}, and thrown (unless 390 * {@code 'skipDontThrowIfBadSRCAttr'} is {@code FALSE}). 391 */ 392 public static Request buildFromTagNodeIter 393 (Iterable<TagNode> source, boolean skipDontThrowIfBadSRCAttr) 394 { 395 Vector<URL> temp = new Vector<>(); 396 int count = 0; 397 Vector<String[]> b64Images = new Vector<>(); 398 Vector<Exception> tagNodeSRCExceptions = new Vector<>(); 399 400 for (TagNode tn : source) 401 { 402 count++; 403 404 if (tn == null) throw new NullPointerException( 405 "The " + count + StringParse.ordinalIndicator(count) + " element of " + 406 "Iterable-Parameter 'source' is null" 407 ); 408 409 String urlStr = tn.AV("src"); 410 411 if (urlStr == null) 412 { 413 SRCException e = new SRCException( 414 "The " + count + StringParse.ordinalIndicator(count) + " TagNode of " + 415 "Iterable-Parameter 'source' does not have a SRC-Attribute" 416 ); 417 418 if (skipDontThrowIfBadSRCAttr) 419 { 420 temp.add(null); 421 b64Images.add(null); 422 tagNodeSRCExceptions.add(e); 423 continue; 424 } 425 426 else throw e; 427 } 428 429 430 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 431 // Handle the Base-64 Stuff, here. 08.31.2023 432 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 433 434 Matcher m = IF.B64_INIT_STRING.matcher(urlStr); 435 436 if (m.find()) 437 { 438 temp.add(null); 439 440 b64Images.add(new String[] { m.group(1), m.group(2)}); 441 442 continue; 443 } 444 445 446 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 447 // Handle the Base-64 Stuff, here. 08.31.2023 448 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 449 450 if (StrCmpr.startsWithNAND(urlStr, "http://", "https://")) 451 { 452 SRCException e = new SRCException( 453 "The " + count + StringParse.ordinalIndicator(count) + " TagNode of " + 454 "Iterable-Parameter 'source' did neither begin with 'http://' nor " + 455 "'https://'. Please build using an 'originalPageURL' parameter to resolve " + 456 "any and all partial URL's." 457 ); 458 459 if (skipDontThrowIfBadSRCAttr) 460 { 461 temp.add(null); 462 b64Images.add(null); 463 tagNodeSRCExceptions.add(e); 464 continue; 465 } 466 467 else throw e; 468 } 469 470 try 471 { temp.add(new URL(urlStr)); } 472 473 catch (MalformedURLException e) 474 { 475 // variable 'e' has already been defined above, use 'e2' 476 SRCException e2 = new SRCException( 477 "When attempting to build the " + count + StringParse.ordinalIndicator(count) + 478 " element of Iterable-Parameter 'source', a MalformedURLException threw. " + 479 "Please see getCause for details.", 480 e 481 ); 482 483 if (skipDontThrowIfBadSRCAttr) 484 { 485 temp.add(null); 486 b64Images.add(null); 487 tagNodeSRCExceptions.add(e2); 488 continue; 489 } 490 491 else throw e2; 492 } 493 } 494 495 return new Request(temp, count, null, b64Images, tagNodeSRCExceptions); 496 } 497 498 /** 499 * Builds an instance of this class using a list of <I><B STYLE='color: red;'>already 500 * prepared</B></I> {@code URL's}. 501 * 502 * @param source <EMBED CLASS='external-html' DATA-FILE-ID=REQUEST_ITER_URL> 503 * 504 * @return A {@code 'Request'} instance. This may be further configured by assigning values to 505 * any / all fields (which will still have their initialized / default-values) 506 * 507 * @throws NullPointerException If any of the {@code URL's} in the {@code Iterable} are null 508 */ 509 public static Request buildFromURLIter(Iterable<URL> source) 510 { 511 Vector<URL> temp = new Vector<>(); 512 int count = 0; 513 514 for (URL url : source) 515 { 516 count++; 517 518 if (url == null) throw new NullPointerException( 519 "The " + count + StringParse.ordinalIndicator(count) + " element of " + 520 "Iterable-Parameter 'source' is null" 521 ); 522 523 temp.add(url); 524 } 525 526 return new Request(temp, count, null); 527 } 528 529 530 // ******************************************************************************************** 531 // ******************************************************************************************** 532 // Package-Visible Utility Methods for ImageScraper, Set by the Constructor. 533 // ******************************************************************************************** 534 // ******************************************************************************************** 535 536 537 // Package-Visibility: Used only in class ImageScraper (to retrieve the Iterable) 538 Iterable<URL> source() { return source; } 539 540 // This Vector-Index Counter is used only once - three lines below 541 private int b64Pos = 0; 542 543 // Package-Visibility: Used only in class Imagescraper (to retrieve a B64-Image String-Array) 544 String[] nextB64Image() 545 { 546 // Since the creation/construction of these Vectors is completely controlled, they should 547 // never be a source of NullPointerException. If for some odd reason they are, it is 548 // better to keep a record indicating that "this really shouldn't have happened" 549 // 550 // These are "assert" statements. There is no reason this method should ever be called 551 // if these are null. In the static-builder, if a null-URL is put into the source-vector 552 // then one of these would be called (b64Images and/or tagNodeSRCExceptions). In such 553 // cases, both of these secondary vectors would already have references put into them. 554 // 555 // Since the "ImageScraper" is heavy-user-interaction class, the paranoia is 10x worse. 556 // This sort of helps mitigate it, although it seems completely superfluous and unnecessary 557 558 if (b64Images == null) throw new UnreachableError(); 559 if (b64Pos >= b64Images.size()) throw new UnreachableError(); 560 561 return b64Images.elementAt(b64Pos++); 562 } 563 564 // This Vector-Index Counter is used only once - three lines below 565 private int tnExPos = 0; 566 567 // Package-Visibility: Used only by class ImageScraper 568 Exception nextTNSRCException() 569 { 570 // Since the creation/construction of these Vectors is completely controlled, they should 571 // never be a source of NullPointerException. If for some odd reason they are, it is 572 // better to keep a record indicating that "this really shouldn't have happened" 573 // 574 // These are "assert" statements. There is no reason this method should ever be called 575 // if these are null. In the static-builder, if a null-URL is put into the source-vector 576 // then one of these would be called (b64Images and/or tagNodeSRCExceptions). In such 577 // cases, both of these secondary vectors would already have references put into them. 578 // 579 // Since the "ImageScraper" is heavy-user-interaction class, the paranoia is 10x worse. 580 // This sort of helps mitigate it, although it seems completely superfluous and unnecessary 581 582 if (tagNodeSRCExceptions == null) throw new UnreachableError(); 583 if (tnExPos >= tagNodeSRCExceptions.size()) throw new UnreachableError(); 584 585 return tagNodeSRCExceptions.elementAt(tnExPos++); 586 } 587 588 589 // ******************************************************************************************** 590 // ******************************************************************************************** 591 // Primary Request Fields 592 // ******************************************************************************************** 593 // ******************************************************************************************** 594 595 596 /** {@code URL} from whence this page has been downloaded */ 597 public final URL originalPageURL; 598 599 // The Source Iterable 600 private final Iterable<URL> source; 601 602 /** The number of Image-{@code URL's} identified inside the {@code 'source'} Iterable. */ 603 public final int size; 604 605 // This is just a zero-padding printer. It adjusts for the number of elements in the original 606 // input Iterable. If there are, for example, under 100 elements, then the first 10 elements 607 // will be padded with a zero. 608 609 final IntFunction<String> counterPrinter; 610 611 // Any & all Base-64 Images. This is usually empty, so it is initialized to null 612 private final Vector<String[]> b64Images; 613 614 // If the user has built from an Iterable<TagNode>, and requested to suppress-exceptions, then 615 // this vector will save those exceptions so that they are ready for the return/result object. 616 617 private final Vector<Exception> tagNodeSRCExceptions; 618 619 620 // ******************************************************************************************** 621 // ******************************************************************************************** 622 // Verbosity & URL-PreProcessor 623 // ******************************************************************************************** 624 // ******************************************************************************************** 625 626 627 /** 628 * Allows a user of this utility to specify how the <B STYLE='color: red;'>Level of 629 * Verbosity</B> (or silence) is applied to the output mechanism while the tool is running. 630 * 631 * <BR /><BR />Note that the Java {@code enum} {@link Verbosity} provides four distint levels, 632 * and that the class {@link ImageScraper} does indeed implement all four variants of textual 633 * output. 634 * 635 * <BR /><BR /><B CLASS=JDDescLabel>{@code NullPointerException}:</B> 636 * 637 * <BR />This field <I><B>may not be null</B></I>, or a {@code NullPointerException}: will 638 * throw! 639 */ 640 public Verbosity verbosity = Verbosity.Normal; 641 642 /** 643 * When non-null, this allows a user to modify any image-{@code URL} immediately-prior to 644 * the {@link ImageScraper} beginning the download process for that image. This is likely of 645 * limited use, but there are certainly situations where (for example) escaped-characters need 646 * to be un-escaped prior to starting the download system. 647 * 648 * <BR /><BR />In such cases, just write a lambda-target that accepts a {@code URL}, and 649 * processes it (in some way, of your chossing), and the downloader will use that updated 650 * {@code URL}-instance for making the HTTP-Connection to download the picture. 651 * 652 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 653 * 654 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 655 * class initializes this field to null. 656 */ 657 public Function<URL, URL> urlPreProcessor = null; 658 659 660 // ******************************************************************************************** 661 // ******************************************************************************************** 662 // Location-Decisions for Saving an Image File (or sending to an 'imageReceiver') 663 // ******************************************************************************************** 664 // ******************************************************************************************** 665 666 667 /** 668 * Allows a user to specify where to save an Image-File after being downloaded. 669 * 670 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 671 * 672 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 673 * class initializes this field to null. 674 */ 675 public Function<ImageInfo, File> targetDirectoryRetriever = null; 676 677 /** 678 * A functional-interface that allows a user to save an image-file to a location of his or her 679 * choosing. Implement this class if saving image files to a target-directory on the 680 * file-system is not acceptable, and the programmer wishes to do something else with the 681 * downloaded images. 682 * 683 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 684 * 685 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 686 * class initializes this field to null. 687 */ 688 public Consumer<ImageInfo> imageReceiver = null; 689 690 /** 691 * When this configuration-field is non-null, this {@code String} parameter is used to 692 * identify the file-system directory to where downloaded image-files are to be saved. 693 * 694 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 695 * 696 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 697 * class initializes this field to null. 698 */ 699 public String targetDirectory = null; 700 701 702 // ******************************************************************************************** 703 // ******************************************************************************************** 704 // File-Name given to an Image File 705 // ******************************************************************************************** 706 // ******************************************************************************************** 707 708 709 /** 710 * When this field is non-null, this {@code String} will be <I>prepended</I> to each image 711 * file-name that is saved or stored to the file-system. 712 * 713 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 714 * 715 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 716 * class initializes this field to null. 717 */ 718 public String fileNamePrefix = null; 719 720 /** 721 * When true, images will be saved according to a counter; when this is {@code FALSE}, the 722 * software will attempt to save these images using their original filenames - picked from 723 * the <B>URL</B>. Saving using a counter is the default behaviour for this class. 724 */ 725 public boolean useDefaultCounterForImageFileNames = true; 726 727 /** 728 * When this field is non-null, each time an image is written to the file-system, this function 729 * will be queried for a file-name before writing the the image-file. 730 * 731 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 732 * 733 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 734 * class initializes this field to null. 735 */ 736 public Function<ImageInfo, String> getImageFileSaveName = null; 737 738 739 // ******************************************************************************************** 740 // ******************************************************************************************** 741 // BOOLEANS'S: Continuing or Throwing on Failure & Exception 742 // ******************************************************************************************** 743 // ******************************************************************************************** 744 745 746 /** 747 * Requests that the downloader-logic catch any & all exceptions that are thrown when 748 * downloading images from an Internet-{@code URL}. The failed download is simply reflected in 749 * the {@link Results} output arrays, and the download-process moves onto the next 750 * {@code Iterable}-Element. 751 * 752 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 753 * 754 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 755 * that are thrown while Java's {@code ImageIO} class is downloading and image, and suddenly 756 * fails. 757 */ 758 public boolean skipOnDownloadException = false; 759 760 /** 761 * Requests that the downloader-logic catch any & all exceptions that are thrown when 762 * decoding Base-64 Encoded Images. The failed conversion is simply reflected in the 763 * {@link Results} output arrays, and the download-process moves onto the next 764 * {@code Iterable}-Element. 765 * 766 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 767 * 768 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 769 * that are thrown when Java's Base-64 Image-Decoder throws an exception. 770 */ 771 public boolean skipOnB64DecodeException = false; 772 773 /** 774 * Requests that the downloader-logic catch any & all exceptions that are thrown while 775 * waiting for an image to finish downloading from an Internet-{@code URL}. The failed 776 * download is simply reflected in the {@link Results} output arrays, and the download-process 777 * moves onto the next {@code Iterable}-Element. 778 * 779 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 780 * 781 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 782 * that are thrown when the Monitor-Thread has timed-out. 783 */ 784 public boolean skipOnTimeOutException = false; 785 786 /** 787 * There are occasions when Java's {@code ImageIO} class returns a null image, rather than 788 * throwing an exception at all. In these cases, the {@link ImageScraper} class throws its own 789 * exception - unless this {@code boolean} has expressly requested to skip-and-move-on when the 790 * {@code ImageIO} returns null from downloading a {@code URL}. 791 * 792 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 793 * 794 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 795 * that are thrown by the {@link ImageScraper} when a downloaded image is null. 796 */ 797 public boolean skipOnNullImageException = false; 798 799 /** 800 * If an attempt is made to write an Image to the File-System, and an exception is thrown, this 801 * boolean requests that rather than throwing the exception, the downloader make a note in the 802 * log that a failure occured, and move on to the next image. 803 * 804 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 805 * 806 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 807 * that are thrown when writing an already downloaded and converted image to the file-system. 808 */ 809 public boolean skipOnImageWritingFail = false; 810 811 /** 812 * This can be helpful if there are any "doubts" about the quality of the Functional-Interfaces 813 * that have been provided to this {@code Request}-instance. 814 * 815 * <BR /><BR /><B CLASS=JDDescLabel>Exception's Skipped:</B> 816 * 817 * <BR />This particular configuration-{@code boolean} allows a user to focus on exceptions 818 * that are thrown by any of the Lambda-Target / Functional-Interfaces that are provided by the 819 * user via this {@code Request}-instance. 820 */ 821 public boolean skipOnUserLambdaException = false; 822 823 824 // ******************************************************************************************** 825 // ******************************************************************************************** 826 // USER-PREDICATE'S & BOOLEAN'S: Which Image Files to Save, and Which to Skip 827 // ******************************************************************************************** 828 // ******************************************************************************************** 829 830 831 /** 832 * If this field is non-null, then before any {@code URL} is connected for a download, the 833 * downloaded mechanism will ask this {@code URL-Predicate} for permission first. If this 834 * {@code Predicate} returns {@code FALSE} for a given {@code URL}, then that image will not be 835 * downloaded, but rather skipped, instead. 836 * 837 * <BR /><BR /><B CLASS=JDDescLabel>Setting to null:</B> 838 * 839 * <BR />This field may be null, and when it is, it shall be ignored. Upon construction, this 840 * class initializes this field to null. 841 */ 842 public Predicate<URL> skipURL = null; 843 844 /** 845 * This scraper has the ability to decode and save {@code Base-64} Images, and they may be 846 * downloaded or skipped - <I>based on this {@code boolean}</I>. If an 847 * {@code Iterable<TagNode>} is passed to the constructor, and one of those 848 * {@code TagNode's} contain an Image Element 849 * ({@code <IMG SRC="data:image/jpeg;base64,...data">}) this class has the ability to 850 * interpret and save the image to a regular image file. By default, {@code Base-64} 851 * images are skipped, but they can also be downloaded as well. 852 */ 853 public boolean skipBase64EncodedImages = false; 854 855 /** 856 * Allows for a user-provided decision-predicate about whether to retain & save, or 857 * discard, an image after downloading. All information available in data-flow class 858 * {@link ImageInfo} is provided to this predicate, and ought to be enough to decide whether 859 * or not to save or reject any of the downloaded image. 860 */ 861 public Predicate<ImageInfo> keeperPredicate = null; 862 863 864 // ******************************************************************************************** 865 // ******************************************************************************************** 866 // Avoiding Hangs and Locks with a TimeOut 867 // ******************************************************************************************** 868 // ******************************************************************************************** 869 870 871 /** 872 * This is the default maximum wait time for an image to download ({@value}). This value may 873 * be reset or modified by instantiating a {@code ImageScraper.AdditionalParameters} class, and 874 * passing the desired values to the constructor. This value is measured in units of 875 * {@code public static final java.util.concurrent.TimeUnit MAX_WAIT_TIME_UNIT} 876 * 877 * @see #MAX_WAIT_TIME_UNIT 878 * @see #maxDownloadWaitTime 879 */ 880 public static final long MAX_WAIT_TIME = 10; 881 882 /** 883 * This is the default measuring unit for the {@code static final long MAX_WAIT_TIME} member. 884 * This value may be reset or modified by instantiating a 885 * {@code ImageScraper.AdditionalParameters} class, and passing the desired values to the 886 * constructor. 887 * 888 * @see #MAX_WAIT_TIME 889 * @see #waitTimeUnits 890 */ 891 public static final TimeUnit MAX_WAIT_TIME_UNIT = TimeUnit.SECONDS; 892 893 /** 894 * If you do not want the downloader to hang on an image, which is sometimes an issue 895 * depending upon the site from which you are making a request, set this parameter, and the 896 * downloader will not wait past that amount of time to download an image. The default 897 * value for this parameter is {@code 10 seconds}. If you do not wish to set the 898 * max-wait-time "the download time-out" counter, then leave the parameter 899 * {@code "waitTimeUnits"} set to {@code null}, and this parameter will be ignored. 900 */ 901 public long maxDownloadWaitTime = MAX_WAIT_TIME; 902 903 /** 904 * This is the "unit of measurement" for the field {@code long maxDownloadWaitTime}. 905 * <BR /><BR /><B>NOTE:</B> <I>This parameter may be {@code null}, and if it is 906 * <SPAN STYLE="color: red;"> both <B>this</B> parameter and the parameter <B>{@code long 907 * maxDownloadWaitTime}</B> will be ignored</SPAN></I>, and the default maximum-wait-time 908 * (download time-out settings) will be used instead. 909 * 910 * <BR /><BR /><B>READ:</B> java.util.concurrent.*; package, and about the {@code class 911 * java.util.concurrent.TimeUnit} for more information. 912 */ 913 public TimeUnit waitTimeUnits = MAX_WAIT_TIME_UNIT; 914 915 916 // ******************************************************************************************** 917 // ******************************************************************************************** 918 // USER AGENT 919 // ******************************************************************************************** 920 // ******************************************************************************************** 921 922 923 /** 924 * There are web-sites that expect a <B STYLE='color: red;'>User-Agent</B> to be defined before 925 * allowing an image download to progress. There are even web-sites and servers that simply 926 * will not connect to a scraper unless a User-Agent is defined. 927 * 928 * <BR /><BR />This is the default <B STYLE='color: red;'>User-Agent</B> that is defined inside 929 * of class {@link Scrape}. 930 * 931 * @see Scrape#USER_AGENT; 932 */ 933 public static final String DEFAULT_USER_AGENT = Scrape.USER_AGENT; 934 935 /** 936 * This 937 */ 938 public String userAgent = DEFAULT_USER_AGENT; 939 940 public boolean alwaysUseUserAgent = false; 941 942 public boolean retryWithUserAgent = true; 943 944 945 // ******************************************************************************************** 946 // ******************************************************************************************** 947 // Check for Validity Method 948 // ******************************************************************************************** 949 // ******************************************************************************************** 950 951 952 void CHECK() 953 { 954 // This is the same as an assert-statement 955 if (source == null) throw new UnreachableError(); 956 957 if (verbosity == null) throw new NullPointerException( 958 "Verbosity has been set to null. It is initialized to Verbosity.Normal. Any level " + 959 "may be chosen, but null is not allowed here." 960 ); 961 962 if (maxDownloadWaitTime < 0) throw new IllegalArgumentException( 963 "You have passed a negative number for parameter maxDownloadWaitTime, and this is " + 964 "not allowed here." 965 ); 966 967 if (waitTimeUnits == null) throw new NullPointerException 968 ("Field 'waitTimeUnits' is null"); 969 970 int count = 971 ((targetDirectory == null) ? 0 : 1) + 972 ((imageReceiver == null) ? 0 : 1) + 973 ((targetDirectoryRetriever == null) ? 0 : 1); 974 975 if (count != 1) throw new IllegalArgumentException( 976 "Of the three Save-Specifiers: targetDirectory, imageReceiver and " + 977 "targetDirectoryRetriever, precisely one of them must be non-null. Upon completion " + 978 "of the class 'Request' Constructor, there are [" + count + "] which are non-null. " + 979 "This is not allowed." 980 ); 981 982 if (targetDirectory != null) 983 { 984 // Ensures that the target directory exists, and is writable 985 WritableDirectoryException.check(targetDirectory); 986 987 // Makes sure that the directory ends with a slash / file-separator. 988 if (! targetDirectory.endsWith(File.separator)) if (targetDirectory.length() > 0) 989 targetDirectory = targetDirectory + File.separator; 990 } 991 } 992 993 994 // ******************************************************************************************** 995 // ******************************************************************************************** 996 // TURN ON **ALL** Exception-Skip Methods 997 // ******************************************************************************************** 998 // ******************************************************************************************** 999 1000 1001 /** 1002 * This allows a user to quickly / easily set all {@code 'skipOn'} flags in one method call. 1003 */ 1004 public void skipOnAllExceptions() 1005 { 1006 // exceptions thrown by Java's ImageIO class when downloading and image 1007 skipOnDownloadException = 1008 1009 // if Java's Base-64 Image-Decoder throws an exception. 1010 skipOnB64DecodeException = 1011 1012 // exception that's thrown when the Monitor-Thread has timed-out. 1013 skipOnTimeOutException = 1014 1015 // exception that's thrown when a downloaded image is null. 1016 skipOnNullImageException = 1017 1018 // exceptions thrown when writing an already downloaded image to the file-system. 1019 skipOnImageWritingFail = 1020 1021 // exceptions thrown by any of the User-Provided Lambda-Target / Functional-Interfaces 1022 skipOnUserLambdaException = true; 1023 } 1024 1025 1026 // ******************************************************************************************** 1027 // ******************************************************************************************** 1028 // Standard-Java Object Methods 1029 // ******************************************************************************************** 1030 // ******************************************************************************************** 1031 1032 1033 private String OBJ_TO_CLASS_NAME(Object o) 1034 { 1035 if (o == null) return "null\n"; 1036 1037 Class<?> c = o.getClass(); 1038 1039 return c.isAnonymousClass() ? "Anonymous Class\n" : (c.getName() + '\n'); 1040 } 1041 1042 private static final String I4 = " "; 1043 1044 /** 1045 * Converts {@code 'this'} instance into a simple Java-{@code String} 1046 * @return A {@code String} where each field has had a 'best efforts' {@code String}-Conversion 1047 */ 1048 public String toString() 1049 { 1050 return 1051 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1052 // Primary Request Fields 1053 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1054 1055 "Primary Request Fields:\n" + 1056 1057 // public final URL originalPageURL; 1058 I4 + "originalPageURL: " + this.originalPageURL + '\n' + 1059 1060 // private final int size; 1061 I4 + "size: " + this.size + '\n' + 1062 1063 // public Verbosity = Verbosity.Normal; 1064 I4 + "Verbosity: " + this.verbosity + '\n' + 1065 1066 '\n' + 1067 1068 1069 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1070 // Location-Decisions for Saving an Image File 1071 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1072 1073 "Location-Decisions for Saving an Image File:\n" + 1074 1075 // public Function<ImageInfo, File> targetDirectoryRetriever = null; 1076 I4 + "targetDirectoryRetriever: " + 1077 OBJ_TO_CLASS_NAME(this.targetDirectoryRetriever) + 1078 1079 // public Consumer<ImageInfo> imageReceiver = null; 1080 I4 + "imageReceiver: " + OBJ_TO_CLASS_NAME(this.imageReceiver) + 1081 1082 // public String targetDirectory = null; 1083 I4 + "targetDirectory: " + ((this.targetDirectory != null) 1084 ? ("[\"" + this.targetDirectory + "\"]") : "null") + '\n' + 1085 1086 '\n' + 1087 1088 1089 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1090 // File-Name given to an Image File 1091 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1092 1093 "File-Name given to an Image File:\n" + 1094 1095 // public String fileNamePrefix = null; 1096 I4 + "fileNamePrefix: " + ((this.fileNamePrefix != null) 1097 ? ("[\"" + this.fileNamePrefix + "\"]") : "null") + '\n' + 1098 1099 // public boolean useDefaultCounterForImageFileNames = true; 1100 I4 + "useDefaultCounterForImageFileNames: " + 1101 this.useDefaultCounterForImageFileNames + '\n' + 1102 1103 // public Function<ImageInfo, String> getImageFileSaveName = null; 1104 I4 + "getImageFileSaveName: " + 1105 OBJ_TO_CLASS_NAME(this.getImageFileSaveName) + '\n' + 1106 1107 '\n' + 1108 1109 1110 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1111 // BOOLEAN'S: Continuing or Throwing on Failure & Exception 1112 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1113 1114 "Boolean-Flags: Continuing or Throwing on Failure & Exception:\n" + 1115 1116 // public boolean skipOnDownloadException = false; 1117 I4 + "skipOnDownloadException: " + this.skipOnDownloadException + '\n' + 1118 1119 // public boolean skipOnB64DecodeException = false; 1120 I4 + "skipOnB64DecodeException: " + this.skipOnB64DecodeException + '\n' + 1121 1122 // public boolean skipOnTimeOutException = false; 1123 I4 + "skipOnTimeOutException: " + this.skipOnTimeOutException + '\n' + 1124 1125 // public boolean skipOnNullImageException = false; 1126 I4 + "skipOnNullImageException: " + this.skipOnNullImageException + '\n' + 1127 1128 // public boolean skipOnImageWritingFail = false; 1129 I4 + "skipOnImageWritingFail: " + this.skipOnImageWritingFail + '\n' + 1130 1131 // public boolean skipOnUserLambdaException = false; 1132 I4 + "skipOnUserLambdaException: " + this.skipOnUserLambdaException + '\n' + 1133 1134 '\n' + 1135 1136 1137 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1138 // USER-PREDICATE'S & BOOLEAN'S: Which Image Files to Save, and Which to Skip 1139 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1140 1141 "User-Predicate's: Which Image-Files to Save, and Which to Skip:\n" + 1142 1143 // public Predicate<URL> skipURL = null; 1144 I4 + "skipURL: " + OBJ_TO_CLASS_NAME(this.skipURL) + 1145 1146 // public boolean skipBase64EncodedImages = false; 1147 I4 + "skipBase64EncodedImages: " + this.skipBase64EncodedImages + '\n' + 1148 1149 // public Predicate<ImageInfo> keeperPredicate = null; 1150 I4 + "keeperPredicate: " + 1151 OBJ_TO_CLASS_NAME(this.keeperPredicate) + 1152 1153 '\n' + 1154 1155 1156 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1157 // Avoiding Hangs and Locks with a TimeOut 1158 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1159 1160 "Avoiding Hangs and Locks with a TimeOut:\n" + 1161 1162 // public long maxDownloadWaitTime = 0; 1163 I4 + "maxDownloadWaitTime: " + this.maxDownloadWaitTime + '\n' + 1164 1165 // public TimeUnit waitTimeUnits = null; 1166 I4 + "waitTimeUnits: " + this.waitTimeUnits + '\n' + 1167 1168 '\n' + 1169 1170 1171 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1172 // USER AGENT 1173 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1174 1175 "Using a User-Agent:\n" + 1176 1177 // public String userAgent = DEFAULT_USER_AGENT; 1178 I4 + "userAgent: " + this.userAgent + '\n' + 1179 1180 // public boolean alwaysUseUserAgent = false; 1181 I4 + "alwaysUseUserAgent: " + this.alwaysUseUserAgent + '\n' + 1182 1183 // public boolean retryWithUserAgent = true; 1184 I4 + "retryWithUserAgent: " + this.retryWithUserAgent + '\n'; 1185 } 1186 1187 1188 // ******************************************************************************************** 1189 // ******************************************************************************************** 1190 // Clone & Clone-Constructor 1191 // ******************************************************************************************** 1192 // ******************************************************************************************** 1193 1194 1195 /** 1196 * Builds a clone of {@code 'this'} instance 1197 * 1198 * @return The copied instance. Note that this is a <B STYLE='color: red;'>shallow</B> clone, 1199 * rather than a <B STYLE='color: red;'>deep</B> clone. The references within the returned 1200 * instances are <I>the exact same references as are in {@code 'this'} instance</I>. 1201 */ 1202 public Request clone() 1203 { return new Request(this); } 1204 1205 // Used by 'clone()'. This constructor only configures the 'final' fields 1206 private Request(Request r) 1207 { 1208 // public final URL originalPageURL; 1209 this.originalPageURL = r.originalPageURL; 1210 1211 // private final Iterable<String> source; 1212 this.source = r.source; 1213 1214 // private final int size; 1215 this.size = r.size; 1216 1217 // final IntFunction<String> counterPrinter; 1218 this.counterPrinter = r.counterPrinter; 1219 1220 // private final Vector<String[]> b64Images; 1221 this.b64Images = r.b64Images; 1222 1223 // private int b64Pos = 0; 1224 this.b64Pos = r.b64Pos; 1225 1226 // private final Vector<Exception> tagNodeSRCExceptions; 1227 this.tagNodeSRCExceptions = r.tagNodeSRCExceptions; 1228 1229 // private int tnExPos = 0; 1230 this.tnExPos = r.tnExPos; 1231 1232 // public Verbosity = Verbosity.Normal; 1233 this.verbosity = r.verbosity; 1234 1235 1236 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1237 // Location-Decisions for Saving an Image File 1238 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1239 1240 // public Function<ImageInfo, File> targetDirectoryRetriever = null; 1241 this.targetDirectoryRetriever = r.targetDirectoryRetriever; 1242 1243 // public Consumer<ImageInfo> imageReceiver = null; 1244 this.imageReceiver = r.imageReceiver; 1245 1246 // public String targetDirectory = null; 1247 this.targetDirectory = r.targetDirectory; 1248 1249 1250 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1251 // File-Name given to an Image File 1252 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1253 1254 // public String fileNamePrefix = null; 1255 this.fileNamePrefix = r.fileNamePrefix; 1256 1257 // public boolean useDefaultCounterForImageFileNames = true; 1258 this.useDefaultCounterForImageFileNames = r.useDefaultCounterForImageFileNames; 1259 1260 // public Function<ImageInfo, String> getImageFileSaveName = null; 1261 this.getImageFileSaveName = r.getImageFileSaveName; 1262 1263 1264 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1265 // BOOLEAN'S: Which Image Files to Save, and Which to Skip 1266 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1267 1268 // public boolean skipOnDownloadException = false; 1269 this.skipOnDownloadException = r.skipOnDownloadException; 1270 1271 // public boolean skipOnB64DecodeException = false; 1272 this.skipOnB64DecodeException = r.skipOnB64DecodeException; 1273 1274 // public boolean skipOnTimeOutException = false; 1275 this.skipOnTimeOutException = r.skipOnTimeOutException; 1276 1277 // public boolean skipOnNullImageException = false; 1278 this.skipOnNullImageException = r.skipOnNullImageException; 1279 1280 // public boolean skipOnImageWritingFail = false; 1281 this.skipOnImageWritingFail = r.skipOnImageWritingFail; 1282 1283 // public boolean skipOnUserLambdaException = false; 1284 this.skipOnUserLambdaException = r.skipOnUserLambdaException; 1285 1286 1287 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1288 // USER-PREDICATE'S & BOOLEAN'S: Which Image Files to Save, and Which to Skip 1289 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1290 1291 // public Predicate<URL> skipURL = null; 1292 this.skipURL = r.skipURL; 1293 1294 // public boolean skipBase64EncodedImages = false; 1295 this.skipBase64EncodedImages = r.skipBase64EncodedImages; 1296 1297 // public Predicate<ImageInfo> keeperPredicate = null; 1298 this.keeperPredicate = r.keeperPredicate; 1299 1300 1301 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1302 // Avoiding Hangs and Locks with a TimeOut 1303 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1304 1305 // public long maxDownloadWaitTime = 0; 1306 this.maxDownloadWaitTime = r.maxDownloadWaitTime; 1307 1308 // public TimeUnit waitTimeUnits = null; 1309 this.waitTimeUnits = r.waitTimeUnits; 1310 1311 1312 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1313 // USER AGENT 1314 // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** 1315 1316 // public String userAgent = DEFAULT_USER_AGENT; 1317 this.userAgent = r.userAgent; 1318 1319 // public boolean alwaysUseUserAgent = false; 1320 this.alwaysUseUserAgent = r.alwaysUseUserAgent; 1321 1322 // public boolean retryWithUserAgent = true; 1323 this.retryWithUserAgent = r.retryWithUserAgent; 1324 } 1325}