package Torello.HTML.Tools.NewsSite;

import Torello.HTML.*;
import Torello.HTML.NodeSearch.*;
import Torello.HTML.Tools.NewsSite.*;
import Torello.Java.*;

import Torello.Languages.LC;

import java.util.*;
import java.util.regex.*;

import java.net.URL;
import java.io.*;

/**
 * This class is nothing more than an 'Example Class' that contains some foreign-language
 * based news web-pages, from both overseas and from Latin America.
 * 
 * <BR /><BR /><EMBED CLASS='external-html' DATA-FILE-ID=NEWS_SITES>
 */
public class NewsSites
{
    private NewsSites() { }

    @SuppressWarnings("unchecked")
    private static final Hashtable<String, Vector<URL>> newsPaperSections = 
        (Hashtable<String, Vector<URL>>) LFEC.readObjectFromFile_JAR
        (NewsSite.class, "data-files/SectionURLs.htdat", true, Hashtable.class);


    /**
     * This example will run the news-site scrape on the Chinese Government News Article
     * Carousel.
     * 
     * <BR /><BR /><B><SPAN STYLE="color: red;">IMPORTANT NOTE:</B></SPAN> This will method will
     * create a directory called <B>"cnb"</B> on your file-system where it will write the contents
     * of (most likely) 15 news-paper articles to disk as HTML files.
     *
     * The output log generated by this method may be viewed here:
     * <BR /><BR /><B><CODE><A HREF='doc-files/Logs/Gov.CN.log.html'>
     * Gov.CN.log.html</A></B></CODE>
     *
     * @throws IOException This throws for IO errors that may occur when reading the web-server,
     * or when saving the web-pages or images to the file-system.
     * 
     * @see FileRW#delTree(String, boolean, Appendable)
     * @see NewsSite
     * @see FileRW#writeFile(CharSequence, String)
     * @see C#toHTML(String, boolean, boolean, boolean)
     */
    public static void runExample() throws IOException
    {
        StorageWriter log = new StorageWriter();

        // This directory will contain ".dat" files that are simply "Serialized" HTML Vectors.
        // Each ".dat" file will contain precisely one HTML page.

        final String dataFilesDir = "cnb" + File.separator + "articleData" + File.separator;

        // This directory will contain sub-directories with ".html" files (and image-files)
        // for each news-article that is saved / downloaded.

        final String htmlFilesDir = "cnb" + File.separator + "articleHTML" + File.separator;

        // This CLEARS WHATEVE DATA IS CURRENTLY IN THE DIRECTORY (by deleting all its contents)
        // The following code is the same as the UNIX Shell Command:
        // rm -r cnb/articleData/
        // mkdir cnb/articleData

        FileRW.delTree(dataFilesDir, true, log);

        // The following code is the same as the UNIX Shell Command:
        // rm -r cnb/articleHTML/
        // mkdir cnb/articleHTML

        FileRW.delTree(htmlFilesDir, true, log);


        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
        // Previous Download Data Erased (if any), Start today's News-Site Scrape
        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
    
        // Use the "GovCNCarousel" instance that is created in this class as a NewsSite
        NewsSite ns = NewsSites.GovCNCarousel;

        // Call the "Scrape URLs" class to retrieve all of the available newspaper articles
        // on the Java-Script "Article Carousel"  Again, the "Article Carousel" is just this 
        // little widget at the top of the page that rotates (usually) five hilited / emphasized
        // news-article links for today

        Vector<Vector<String>> articleURLs = ScrapeURLs.get(ns, log);

        // This is usually not very important if only a small number of articles are being
        // scraped.  When downloading hundreds of articles - being able to pause if there is a
        // web-site IOError (And restart) is very important.
        //
        // The standard factory-generated "getFSInstance" creates a small file on the file-system
        // for saving the "Download State" while downloading...
    
        Pause pause = Pause.getFSInstance("cnb" + File.separator + "state.dat");
        pause.initialize();

        // The "Scraped Articles" will be sent to the directory named by "dataFilesDir"
        // Using the File-System to save these articles is the default-factory means for
        // saving article-data.  Writing a customized "ScapedArticleReceiver" to do anything
        // from saving article-data to a Data-Base up to and including e-mailing article data
        // is possible using a self-written "ScrapedArticleReceiver"

        ScrapedArticleReceiver receiver = ScrapedArticleReceiver.saveToFS(dataFilesDir);

        // This will download each of the article's from their web-page URL.  The web-page
        // article URL's were retrieved by "Scraped URLs".  The saved HTML (as HTML Vectors)
        // is sent to the "Article Receiver" (defined in the previous step).  These news articles
        // are saved as ".dat" since they are serialized java-objects.
        //
        // Explaining some "unnamed parameters" passed to the method invocation below:
        //
        // true: [skipArticlesWithoutPhotos] Skips Mandarin Chinese Newspaper Articles that do not
        //       include at least one photo.  Photos usually help when reading foreign news articles.
        // null: [bannerAndAdFinder] Some sites include images for Facebook links or advertising.
        //       Gov.CN usually doesn't have these, but occasionally there are extraneous links.
        //       for the purposes of this example, this parameter is ignored, and passed null.
        // false: [keepOriginalPageHTML] The "Complete Page" - content before the Article Body is
        //        extracted from the Article Web-Page is not saved.  This can occasionally be useful
        //        if the HTML <HEAD>...</HEAD> has JSON or React-JS data to extract.

        ScrapeArticles.download
            (receiver, articleURLs, ns.articleGetter, true, null, false, pause, log);
        
        // Now this will convert each of the ".dat" files to an ".html" file - and also it
        // will download the pictures / image included in the article.
        //
        // Explaining some "unnamed parameters" passed to the method invocation below:
        //
        // true: [cleanIt] This runs some basic HTML remove operations.  The best way to see
        //       what the parameter "cleanIt" asks to have removed is to view the class "ToHTML"
        // null: [HTMLModifier] Cleaning up other extraneous links and content in an newspaper
        //       article body like advertising or links to other articles is usually necessary.
        //       Anywhere between 1 and 10 lines of NodeSearch Removal Operations will get rid of
        //       unnecessary HTML.  For the purposes of this example, such a cleaning operation is
        //       not done here - although the final articles do include some "links to other
        //       articles" that is not "CLEANED" like it should be.

        ToHTML.convert(dataFilesDir, htmlFilesDir, true, null, log);

        // NOTE: The log of running this command on Debian UNIX / LINUX may be viewed in the
        // JavaDoc Comments in the top of this method.  If this method is run in an MS-DOS
        // or Windows Environment, there will be no screen colors available to view.

        FileRW.writeFile(
            C.toHTML(log.getString(), true, true, true),
            "cnb" + File.separator + "Gov.CN.log.html"
        );
    }

    /**
     * Prints the contents of the Data File.  Invoking this command allows a programmer to see
     * which "sub-sections" are ascribed to each of the different news-paper definitions in this
     * class.  Each "sub-section" is nothing more than a {@code URL}-branch of the primary web
     * site {@code URL}.
     *
     * <DIV CLASS="HTML">{@code
     * <!-- If the following were the primary news-site -->
     * http://news.baidu.com
     * 
     * <!-- This would be a "sub-section" of the primary site -->
     * http://news.baidu.com/sports
     * }</DIV>
     *
     * <BR /><BR />Can be called from the command line.
     * <BR /><BR />If a single command-line argument is passed to {@code "argv[0]"}, the contents
     * of the "Sections URL Data File" will be output to a text-file that is named using the
     * {@code String} passed to {@code "argv[0]"}.
     *
     * @param argv These are the command line arguments passed by the JRE to this method.
     * @throws IOException If there are any problems while attempting to save the output to the
     * the output file (if one was named / requested).
     */
    public static void main(String[] argv) throws IOException
    {
        // Uncomment this line to run the example code (instead of section-data print)
        // runExample(); System.exit(0);

        // The data-file is loaded into private field "newsPaperSections"
        // This private field is a Hashtable<String, Vector<URL>>.  Convert each of
        // these sections so that they may be printed to terminal and maybe to a text
        // file.

        StringBuilder sb = new StringBuilder();

        for (String newspaper : newsPaperSections.keySet())
        {
            sb.append(newspaper + '\n');
            for (URL section : newsPaperSections.get(newspaper))
                sb.append(section.toString() + '\n');
            sb.append("\n\n***************************************************\n\n");
        }
        
        String s = sb.toString();
        System.out.println(s);
        
        // If there is a command-line parameter, it shall be interpreted a file-name.
        // The contents of the "sections data-file" (as text) will be written a file on the
        // file-system using the String-value of "argv[0]" as the name of the output-filename.

        if (argv.length == 1) FileRW.writeFile(s, argv[0]);
    }

    // URLFilter.regexKEEP(Pattern.compile("^http.+baidu\\.com\\/s\\?id=\\d+$")));
    // ArticleGet.usual(TextComparitor.CN_CI, "article-content"));

    /**
     * The News Site at address: <CODE><A HREF="https://www.abc.es/" TARGET=_blank>
     * "https://www.abc.es/"</A></CODE> is slightly more complicated when retrieving News-Article
     * Links.
     *
     * <BR /><BR />Notice that each newspaper article {@code URL}-link is "wrapped" in an HTML
     * {@code '<ARTICLE>...</ARTICLE>'} Element.
     *
     * <BR /><BR />If this code were translated into an "XPath Query" or "CSS Selector", it would
     * read: <B>{@code article a}</B>.  Specifically it says to find all {@code 'Anchor'} elements
     * that are descendants of {@code 'Article'} Elements.
     * 
     * @see TagNodeFindL1Inclusive#all(Vector, String)
     * @see TagNodeGet#first(Vector, int, int, TC, String[])
     * @see TagNode#AV(String)
     */
    public static Vector<String> ABC_LINKS_GETTER(URL url, Vector<HTMLNode> page)
    {
        Vector<String> ret = new Vector<>();       TagNode tn;     String urlStr;

        // Links are kept inside <ARTICLE> ... </ARTICLE> on the main / section page.
        for (DotPair article : TagNodeFindL1Inclusive.all(page, "article"))

            // Now find the <A HREF=...> ... </A>
            if ((tn = TagNodeGet.first(page, article.start, article.end, TC.OpeningTags, "a"))
                != null)

                if ((urlStr = tn.AV("href")) != null)
                    ret.add(urlStr);

        return ret;
    }

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.abc.es/" TARGET=_blank>https://www.abc.es/</A></CODE>.
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>ABC España</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>Spain</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://abc.es}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Spanish</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link StrFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD>{@code 'HREF'} must end with {@code '.html'}
     *          <BR />See: {@link StrFilter#comparitor(TextComparitor, String[])}
     *          <BR />See: {@link TextComparitor#EW_CI}
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD>Invokes method {@link #ABC_LINKS_GETTER(URL, Vector)}</TD>
     * </TR>
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <MAIN>...</MAIN>}<BR />See: {@link ArticleGet#usual(String)}</TD>
     * </TR>
     * </TABLE>
     * 
     * <BR />View a copy of the logs that are generated from using this {@code NewsSite} instance.
     * <BR /><BR /><UL CLASS=JDUL>
     * <LI> <CODE><B><A HREF='doc-files/Logs/ABC.ES-ScrapeURLs.html'>
     *      ABC.ES ScrapeURLs LOG</A></B></CODE>
     *      </LI>
     * <LI> {@code ScrapeArticles}
     *      <BR /><B>IMPORTANT NOTE:</B> Though {@code ScrapeURL's} code <I>will check for
     *      duplicate {@code URL's}</I> that may be returned <I>within any given-section</I>,
     *      {@code Article URL's} may be repeated among the different sections of the newspaper.
     *      Since the {@code URL}-scrape returned nearly 3,000 articles, the log of
     *      an {@code Article} scrape is not included here.  Proper duplicate {@code URL} checking
     *      code has obviously been written, but would be too complicated to show in this example.
     *     </LI>
     * </UL>
     * 
     * <EMBED CLASS='external-html' DATA-FILE-ID=NEWS_STE_CHANGE>
     */
    public static final NewsSite ABCES = new NewsSite
    (
        "ABC España", Country.Spain, "https://www.abc.es/", LC.ES,
        "ABC is a Spanish national daily newspaper.  It is the third largest general-interest " +
        "newspaper in Spain, and the oldest newspaper still operating in Madrid.",
        newsPaperSections.get("ABCES"),
        StrFilter.comparitor(TextComparitor.EW_CI, ".html"),
        NewsSites::ABC_LINKS_GETTER,
        ArticleGet.usual("main"),
        null /* bannerAndAdFinder */
    );

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.elpulso.mx/" TARGET=_blank>https://www.elpulso.mx/</A></CODE>.
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>El Pulso, México</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>México</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://elpulso.mx}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Spanish</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link StrFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD>{@code HREF} must match: {@code http://some.domain/YYYY/MM/DD/<article-name>/}</TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD><B>{@code null}</B>.  Retrieves <B><I>all</I></B> Anchor-Links on a Section-Page.
     *          Note that {@code URL's} must still pass the previous {@code StrFilter} (above)
     *          in order to be parsed as {@link Article}'s.
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <DIV CLASS="entry-content">...</DIV>}
     *          <BR />See: {@link ArticleGet#usual(TextComparitor, String[])}
     *          <BR />See: {@link TextComparitor#C}
     *      </TD>
     * </TR>
     * </TABLE>
     */
    public static final NewsSite Pulso = new NewsSite
    (
        "El Pulso, México", Country.Mexico, "https://elpulso.mx", LC.ES,
        "El Pulso newspaper is Spanish language newspaper in Mexico. It is showing breaking " +
            "news, " +
        "headlines, kids news, tourism news, entertainment news, study news, industrial news, " +
        "economical news, health & beauty news, crime news, career news, Travel news, " +
        "diet & fitness news, Top stories, special news, celebrity news.",
        newsPaperSections.get("PULSO"),
        StrFilter.regExKEEP(Pattern.compile(
            "^https?:\\/{2}.*?\\/\\d{4}\\/\\d{2}\\/\\d{2}\\/[\\w-]{10,}\\/$"
        ), false),
        null /* LinksGet */,
        ArticleGet.usual(TextComparitor.C, "entry-content"),
        null /* bannerAndAddFinder */
    );

    /**
     * The News Site at address: <CODE><A HREF="https://www.ElNacional.com/" TARGET=_blank>
     * "https://www.ElNacional.com/"</A></CODE> is slightly more complicated when retrieving
     * News-Article Links.
     *
     * <BR /><BR />Notice that each newspaper article {@code URL}-link is "wrapped" in an HTML
     * {@code '<DIV CLASS="td-module-thumb">...</DIV>'} Element.
     *
     * <BR /><BR />If this code were translated into an "XPath Query" or "CSS Selector", it would
     * read: <B>{@code div.td-module-thumb a}</B>.  Specifically it says to find all
     * {@code 'Anchor'} elements that are descendants of {@code 'DIV'} Elements where said
     * Divider's CSS {@code CLASS} contains {@code 'td-module-thumb'}.
     * 
     * @see InnerTagFindInclusive#all(Vector, String, String, TextComparitor, String[])
     * @see TagNodeGet#first(Vector, int, int, TC, String[])
     * @see TagNode#AV(String)
     */
    public static Vector<String> EL_NACIONAL_LINKS_GETTER(URL url, Vector<HTMLNode> page)
    {
        Vector<String> ret = new Vector<>();       TagNode tn;     String urlStr;

        // Links are kept inside <DIV CLASS=td-module-thumb> ... </DIV> on the main / section page.
        for (DotPair article : InnerTagFindInclusive.all
            (page, "div", "class", TextComparitor.C, "td-module-thumb"))

            // Now find the <A HREF=...> ... </A>
            if ((tn = TagNodeGet.first
                (page, article.start, article.end, TC.OpeningTags, "a")) != null)

                if ((urlStr = tn.AV("href")) != null)
                    ret.add(urlStr);

        return ret;
    }

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.elnacional.com/" TARGET=_blank>
     * https://www.elnacional.com/</A></CODE>.
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>El Nacional</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>Venezuela</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://elnacional.com}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Spanish</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link URLFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD><B>{@code null}</B>.  The {@code LinksGet} provided here will only return valid
     *          {@code Article URL's}, so there is no need for a {@code URLFilter}.
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD>Invokes method {@link #EL_NACIONAL_LINKS_GETTER(URL, Vector)}</TD>
     * </TR>
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <ARTICLE>...</ARTICLE>}<BR />See: {@link ArticleGet#usual(String)}</TD>
     * </TR>
     * </TABLE>
     * 
     * <BR />View a copy of the logs that are generated from using this {@code NewsSite}.
     * <BR /><BR /><UL CLASS=JDUL>
     * <LI> <CODE><B><A HREF='doc-files/Logs/ElNacional-ScrapeURLs.html'>
     *      El Nacional ScrapeURLs LOG</A></B></CODE>
     *      </LI>
     * <LI> <CODE><B><A HREF='doc-files/Logs/ElNacional-ScrapeArticles.html'>
     *      El Nacional ScrapeArticles LOG</A></B></CODE>
     *      </LI>
     * </UL>
     * 
     * <EMBED CLASS='external-html' DATA-FILE-ID=NEWS_STE_CHANGE>
     */
    public static final NewsSite ElNacional = new NewsSite
    (
        "El Nacional", Country.Venezuela, "https://elnacional.com", LC.ES,
        "El Nacional is a Venezuelan publishing company under the name C.A. Editorial " +
            "El Nacional, " +
        "most widely known for its El Nacional newspaper and website. It, along with Últimas " +
        "Noticias and El Universal, are the most widely read and circulated daily national " +
        "newspapers in the country, and it has an average of more than 80,000 papers distributed " +
        "daily and 170,000 copies on weekends.",
        newsPaperSections.get("ElNacional"),
        (URLFilter) null, /* The LinksGetter will only return valid Anchor's */
        NewsSites::EL_NACIONAL_LINKS_GETTER,
        ArticleGet.usual("article"),
        null /* bannerAndAdFinder */
    );

    /**
     * The News Site at address: <CODE><A HREF="https://www.ElEspectador.com/" TARGET=_blank>
     * "https://www.ElEspectador.com/"</A></CODE> is slightly more complicated when retrieving
     * News-Article Links.
     *
     * <BR /><BR />Notice that each newspaper article {@code URL}-link is "wrapped" in an HTML
     * {@code '<DIV CLASS="Card ...">...</DIV>'} Element.
     *
     * <BR /><BR />If this code were translated into an "XPath Query" or "CSS Selector", it would
     * read: <B>{@code div.Card a.card-link}</B>.  Specifically it says to find all
     * {@code 'Anchor'} elements whose CSS {@code Class} contains {@code 'card-link'} and which
     * are descendants of {@code 'DIV'} Elements where said Divider's
     * CSS {@code CLASS} contains {@code 'Card'}.
     *
     * @see InnerTagFindInclusive#all(Vector, String, String, TextComparitor, String[])
     * @see InnerTagGet#first(Vector, int, int, String, String, TextComparitor, String[])
     * @see TagNode#AV(String)
     */
    public static Vector<String> EL_ESPECTADOR_LINKS_GETTER(URL url, Vector<HTMLNode> page)
    {
        Vector<String> ret = new Vector<>();

        TagNode tn;
        String  urlStr;

        // Links are kept inside <DIV CLASS="Card ..."> ... </DIV> on the main / section page.
        for (DotPair article : InnerTagFindInclusive.all
            (page, "div", "class", TextComparitor.C, "Card"))

            // Now find the <A CLASS="card-link" HREF=...> ... </A>
            if ((tn = InnerTagGet.first
                (page, article.start, article.end, "a", "class", TextComparitor.C, "card-link"))
                    != null)

                if ((urlStr = tn.AV("href")) != null)
                    ret.add(urlStr);

        return ret;
    }

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.elespectador.com/" TARGET=_blank>
     * https://www.elespectador.com/</A></CODE>.
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>El Espectador</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>Columbia</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://elespectador.com}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Spanish</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link StrFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD>{@code HREF} must end with a forward-slash {@code '/'} character.
     *          <BR />See: {@link TextComparitor#ENDS_WITH}
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD>Invokes method {@link #EL_NACIONAL_LINKS_GETTER(URL, Vector)}</TD>
     * </TR>
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <DIV CLASS="l-main">...</DIV>}
     *          <BR />See: {@link ArticleGet#usual(TextComparitor, String[])}
     *          <BR />See: {@link TextComparitor#C}
     *      </TD>
     * </TR>
     * </TABLE>
     * 
     * <BR />View a copy of the logs that are generated from using this {@code NewsSite}.
     * <BR /><BR /><UL CLASS=JDUL>
     * <LI> <CODE><B><A HREF='doc-files/Logs/ElEspectador-ScrapeURLs.html'>
     *      El Espectador ScrapeURLs LOG</A></B></CODE>
     *      </LI>
     * <LI> <CODE><B><A HREF='doc-files/Logs/ElEspectador-ScrapeArticles.html'>
     *      El Espectador ScrapeArticles LOG</A></B></CODE>
     *      </LI>
     * </UL>
     * 
     * <EMBED CLASS='external-html' DATA-FILE-ID=NEWS_STE_CHANGE>
     */
    public static final NewsSite ElEspectador = new NewsSite
    (
        "El Espectador, Columbia", Country.Colombia, "https://elespectador.com", LC.ES,
        "El Espectador (meaning \"The Spectator\") is a newspaper with national circulation " +
            "within " +
        "Colombia, founded by Fidel Cano Gutiérrez on 22 March 1887 in Medellín and published " +
        "since 1915 in Bogotá. It changed from a daily to a weekly edition in 2001, following a " +
        "financial crisis, and became a daily again on 11 May 2008, a comeback which had been " +
        "long rumoured, in tabloid format (28 x 39.5 cm). From 1997 to 2011 its main shareholder " +
        "was Julio Mario Santo Domingo.",
        newsPaperSections.get("ElEspectador"),
        StrFilter.comparitor(TextComparitor.ENDS_WITH, "/"),
        NewsSites::EL_ESPECTADOR_LINKS_GETTER,
        ArticleGet.usual("article"),
        null /* bannerAndAdFinder */
    );


    /**
     * The News Site at address: <CODE><A HREF="https://www.gov.cn/" TARGET=_blank>
     * "https://www.gov.cn/"</A></CODE> has a Java-Script "Links Carousel".  Essentially, there
     * is a section with "Showcased News Articles" that are intended to be emphasize anywhere
     * between four and eight primary articles.
     *
     * <BR /><BR />This Links-Carousel is wrapped in an HTML Divider Element as below:
     * {@code <DIV CLASS="slider-carousel">}.
     *
     * <BR /><BR />If this code were translated into an "XPath Query" or "CSS Selector", it would
     * read: <B>{@code div[class=slider-carousel] a}</B>.  Specifically it says to  find all
     * {@code 'Anchor'} elements that are descendants of {@code '<DIV CLASS="slider-carousel">'}
     * Elements.
     * 
     * @see InnerTagGetInclusive#first(Vector, String, String, TextComparitor, String[])
     * @see TagNodeGet#all(Vector, TC, String[])
     * @see TagNode#AV(String)
     */
    public static Vector<String> GOVCN_CAROUSEL_LINKS_GETTER(URL url, Vector<HTMLNode> page)
    {
        Vector<String>  ret     = new Vector<>();
        String          urlStr;

        // Find the first <DIV CLASS="slider-carousel"> ... </DIV> section
        Vector<HTMLNode> carouselDIV = InnerTagGetInclusive.first
            (page, "div", "class", TextComparitor.CN_CI, "slider-carousel");

        // Retrieve any HTML Anchor <A HREF=...> ... </A> found within the contents of the
        // Divider.

        for (TagNode tn: TagNodeGet.all(carouselDIV, TC.OpeningTags, "a"))
            if ((urlStr = tn.AV("href")) != null)
                ret.add(urlStr);

        return ret;
    };

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.gov.cn/" TARGET=_blank>
     * https://www.gov.cn/</A></CODE>.
     * 
     * <BR /><BR />The "Carousels" are just the emphasized or "HiLighted" links that are
     * on three separate pages.  There is a complete-link {@code NewsSite} definition that
     * will retrieve all links - <I>not just the links hilited by the carousel.</I>
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>Chinese Government Web Portal</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>People's Republic of China</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://gov.cn}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Mandarin Chinese</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link StrFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD>{@code HREF} must match:
     *          {@code "^http://www.gov.cn/(?:.+?/)?\\d{4}-\\d{2}/\\d{2}/(?:.+?/)?content_\\d+.htm(?:l)?(#\\d+)?"}
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD>Invokes method {@link #GOVCN_CAROUSEL_LINKS_GETTER(URL, Vector)}</TD>
     * </TR>
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <DIV CLASS="article ...">...</DIV>}
     *          <BR />See: {@link ArticleGet#usual(TextComparitor, String[])}
     *          <BR />See: {@link TextComparitor#C}
     *      </TD>
     * </TR>
     * </TABLE>
     * 
     * <BR />View a copy of the logs that are generated from using this {@code NewsSite}.
     * <BR /><BR /><UL CLASS=JDUL>
     * <LI> <CODE><B><A HREF='doc-files/Logs/GovCNCarousel-ScrapeURLs.html'>
     *      Gov.CN Carousel ScrapeURLs LOG</A></B></CODE>
     *      </LI>
     * <LI> <CODE><B><A HREF='doc-files/Logs/GovCNCarousel-ScrapeArticles.html'>
     *      Gov.CN Carousel ScrapeArticles LOG</A></B></CODE>
     *      </LI>
     * </UL>
     * 
     * <EMBED CLASS='external-html' DATA-FILE-ID=NEWS_STE_CHANGE>
     */
    public static final NewsSite GovCNCarousel = new NewsSite
    (
        "Chinese Government Web Portal", Country.China, "https://gov.cn/", LC.ZH_CN,
        "The Chinese Government Sponsored Web-Site",
        newsPaperSections.get("GovCNCarousel"),
        StrFilter.regExKEEP(Pattern.compile(
            "^http://www.gov.cn/(?:.+?/)?\\d{4}-\\d{2}/\\d{2}/(?:.+?/)?" +
                "content_\\d+.htm(?:l)?(#\\d+)?"
        ), false),
        NewsSites::GOVCN_CAROUSEL_LINKS_GETTER,
        ArticleGet.usual(TextComparitor.C, "article"),
        null /* bannerAndAddFinder */
    );

    /**
     * This is the {@code NewsSite} definition for the Newspaper located at:
     * <CODE><A HREF="https://www.gov.cn/" TARGET=_blank>
     * https://www.gov.cn/</A></CODE>.
     *
     * <BR /><BR />This version of the "Gov.CN" website will scour a larger set of section
     * {@code URL's}, and will not limit the returned Article-Links to just those found on the
     * java-script carousel.  The Java-Script Carousel will almost always have a total of five
     * news-article links available.  This definition of {@code 'NewsSite'} may return up to 
     * thirty to forty different articles per news-section.
     * 
     * <BR /><BR /><TABLE CLASS=JDBriefTable>
     * <TR><TH>Parameter</TH>                       <TH>Significance</TH></TR>
     * <TR><TD>Newspaper Name</TD>                  <TD>Chinese Government Web Portal</TD></TR>
     * <TR><TD>Country of Origin</TD>               <TD>People's Republic of China</TD></TR>
     * <TR><TD>Website URL</TD>                     <TD>{@code https://gov.cn}</TD></TR>
     * <TR><TD>Newspaper Printing Language</TD>     <TD>Mandarin Chinese</TD></TR>
     * </TABLE>
     * 
     * <BR /><TABLE CLASS=NEWSSITE>
     * <TR><TH>Parameter</TH><TH>Purpose</TH><TH>Value</TH></TR>
     * <TR> <TD>Newspaper Article Groups / Sections</TD>
     *      <TD>Scrape Sections</TD>
     *      <TD><I>Retrieved from Data File</I></TD>
     * </TR>
     * <TR> <TD><B>{@link StrFilter}</B></TD>
     *      <TD>News Web-Site Section-Page Aritlce-Link ({@code <A HREF=...>}) Filter</TD>
     *      <TD>{@code HREF} must match:
     *          {@code "^http://www.gov.cn/(?:.+?/)?\\d{4}-\\d{2}/\\d{2}/(?:.+?/)?content_\\d+.htm(?:l)?(#\\d+)?"}
     *      </TD>
     * </TR>
     * <TR> <TD><B>{@link LinksGet}</B></TD>
     *      <TD>Used to <B><I>manually</I></B> retrieve Article-Link {@code URL's}</TD>
     *      <TD><B>{@code null}</B>.  Retrieves <B><I>all</I></B> Anchor-Links on a Section-Page.
     *          Note that {@code URL's} must still pass the previous {@code StrFilter} (above)
     *          in order to be parsed as {@link Article}'s.
     *      </TD>
     * </TR> 
     * <TR> <TD><B>{@link ArticleGet}</B></TD>
     *      <TD>Retrieves Article-Body Content from an Article-Link Web-Page</TD>
     *      <TD>{@code <DIV CLASS="article ...">...</DIV>}
     *          <BR />See: {@link ArticleGet#usual(TextComparitor, String[])}
     *          <BR />See: {@link TextComparitor#C}
     *      </TD>
     * </TR>
     * </TABLE>
     * 
     * <BR />View a copy of the logs that are generated from using this {@code NewsSite}.
     * <BR /><BR /><UL CLASS=JDUL>
     * <LI> <CODE><B><A HREF='doc-files/Logs/GovCN-ScrapeURLs.html'>
     *      Gov.CN ScrapeURLs LOG</A></B></CODE>
     *      </LI>
     * <LI> <CODE><B><A HREF='doc-files/Logs/GovCN-ScrapeArticles.html'>
     *      Gov.CN ScrapeArticles LOG</A></B></CODE>
     *      </LI>
     * </UL>
     * 
     * <EMBED CLASS='external-html' DATA-FILE-ID=NEWS_STE_CHANGE>
     */
    public static final NewsSite GovCN = new NewsSite
    (
        "Chinese Government Web Portal", Country.China, "https://gov.cn/", LC.ZH_CN,
        "The Chinese Government Sponsored Web-Site",
        newsPaperSections.get("GovCN"),
        StrFilter.regExKEEP(Pattern.compile(
            "^http://www.gov.cn/(?:.+?/)?\\d{4}-\\d{2}/\\d{2}/(?:.+?/)?" +
                "content_\\d+.htm(?:l)?(#\\d+)?"
        ), false),
        null,
        ArticleGet.usual(TextComparitor.C, "article"),
        null /* bannerAndAddFinder */
    );
}