001package Torello.Browser;
002
003import java.util.*;
004import javax.json.*;
005import java.io.*;
006
007import java.net.URL;
008import java.util.function.Function;
009import java.util.function.Consumer;
010
011import Torello.Java.*;
012import Torello.Java.Additional.*;
013
014import static Torello.Java.C.*;
015
016import Torello.HTML.Scrape;
017import NeoVisionaries.WebSockets.WebSocketException;
018
019/**
020 * This class helps to start an headless-instance of a Web-Browser, and open a connection to 
021 * that browser.
022 * 
023 * <EMBED CLASS='external-html' DATA-FILE-ID=BRDPC>
024 */
025public class BRDPC
026{
027    private BRDPC() { }
028
029    // ********************************************************************************************
030    // ********************************************************************************************
031    // Looking up the name of all registered events
032    // ********************************************************************************************
033    // ********************************************************************************************
034
035
036    // Converts a Browser-Event name to the **PARTIALLY-QUALIFIED** Event-Name
037    @SuppressWarnings("unchecked")
038    private static final TreeMap<String, String> eventNames = (TreeMap<String, String>) 
039        LFEC.readObjectFromFile_JAR
040            (BRDPC.class, "data-files/EventNames.tmdat", true, TreeMap.class);
041
042    /*
043     * Chrome only provides the name of the event, this will return the partially qualified name
044     * of the class - <B STYLE='color: red;'><I>including the containing / parent class name, 
045     * which is needed to construct an instance of the event-class</I></B>.
046     * 
047     * <BR /><BR /><B>NOTE:</B> The package name <B>({@code Torello.Browser})</B> is not included
048     * in the returned {@code String}.
049     */
050    static final String getEventClassName(String eventName)
051    { return eventNames.get(eventName); }
052
053
054    // ********************************************************************************************
055    // ********************************************************************************************
056    //
057    // ********************************************************************************************
058    // ********************************************************************************************
059
060
061    /** A singleton instance of the web-socket connection. */
062    public static WebSocketSender defaultSender = null;
063
064    // A lambda used by many of the types in this package.
065    static final Function<JsonObject, Ret0> NoReturnValues = (JsonObject jo) -> Ret0.R0;
066
067    /**
068     * A commmon location for the Chrome-Binary on a Windows Machine.
069     * 
070     * <BR /><BR /><B STYLE='color: red;'>IMPORTANT:</B> When this class' 
071     * {@link #startBrowserExecutable(boolean, Integer, String, int)} method is invoked, it
072     * will attempt to run the program specified by this field.  <B><I>It is of utmost importance
073     * to set this field to whatever location has your Chrome-Binary!</I></B>
074     */
075    public static String CHROME_EXE =
076        "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe";
077
078    /**
079     * A common location for the Chome-Canary Binary on a Windows Machine.
080     * 
081     * <BR /><BR /><B STYLE='color: red;'>IMPORANT:</B> I don't actually know when or why one
082     * would use the <B>Canary</B>-Binary, rather than the standard <B>Chrome</B> executable.
083     * I have read that the original intent of this particular application (which is a full 
084     * fledged instance of <B>Chrome</B>) - was to provide a headless version of the standard
085     * Google-Browser.
086     * 
087     * <BR /><BR />The current version of a <B>Chrome</B>-Executable seems to accept
088     * {@code Remote Debug Port} commands quite readily, without having to use <B>Canary</B>.
089     * I will leave it here as a discussion-point.
090     */
091    public static String CANARY_EXE =
092        "C:\\Users\\ralph\\AppData\\Local\\Google\\Chrome SxS\\Application\\Chrome";
093
094    /**
095     * Configuration-flag for setting the verbosity of this library-package.
096     * 
097     * <BR /><BR />To sent more vebose messages to your {@code StorageWriter} instance (field
098     * {@link #sw}), change the value of this field to {@code TRUE}.
099     */
100    public static boolean QUIET = false;
101
102    /**
103     * The log output.  Output text will be sent to this field.  This field is not declared 
104     * {@code final}, and (obviously) may be set to any desired log output.  This field is not
105     * exactly thread-safe - <I>it is shared by all threads which use the headless browser
106     * library tools!</I>
107     * 
108     * <BR /><BR />Because web-sockets, themselves are a multi-threaded application, there isn't
109     * really a way to identify which thread has caused a particular response.  Some percentage of
110     * the messages received from the browser will be events, and in a multi-threaded application
111     * of this tool, there would be no way to identify which thread caused which event.
112     */
113    public static StorageWriter sw = new StorageWriter();
114
115
116    // ********************************************************************************************
117    // ********************************************************************************************
118    // Starting the Browser, from Java rather than the command-line
119    // ********************************************************************************************
120    // ********************************************************************************************
121
122
123    /**
124     * <H3 CLASS=JDBanner>A browser-instance must be loaded before using this package's commands
125     * </H3>
126     * 
127     * <BR />
128     * Starts a headless (or full/complete) browser instance.  All this method does is use the 
129     * Standard-Java {@code java.lang.Process} class to start an operating-system executable.  This
130     * just means invoking the executable specified by {@link #CHROME_EXE}.
131     * 
132     * @param headless If {@code FALSE} is passed to this method, then the option {@code -headless}
133     * will not be passed to the browser at startup.  In such scenarios, an actual browser should
134     * popup on your desktop.  Normally, this parameter should receive {@code TRUE}.
135     * 
136     * @param port  It is standard operating procedure to pass {@code 9222} to this parameter.
137     * {@code 9223} is also very common.  You may pass null to this parameter, and the default
138     * value will be used (which is {@code 9223}).
139     * 
140     * @param initialURL You may elect to have the page open at the specified {@code URL}.  This
141     * parameter may be null, and when it is, it shall be ignored.
142     * 
143     * @param maxCount Some delay is inserted between the starting of a browser executable, and
144     * this method exiting.
145     * 
146     * @throws IOException If there are any web-socket or http problems when attempting to connect
147     * to the browser.
148     */
149    public static void startBrowserExecutable
150        (boolean headless, Integer port, String initialURL, int maxCount)
151        throws IOException
152    {
153        if (port == null) port = 9223;
154
155        if (maxCount < 2) maxCount = 2;
156
157        Vector<String> comm = new Vector<>();
158
159        comm.add(CHROME_EXE);
160
161        if (! headless) comm.add("-disable-gpu");
162
163        comm.add("-remote-debugging-port=" + port);
164
165        if (headless) comm.add("-headless");
166
167        // I don't know as much about Google-Chrome as you might think...
168        // I don't know what this does...
169        // comm.add("-no-sandbox");
170
171        if (initialURL != null) comm.add(initialURL);
172
173        String[] COMM = comm.toArray(new String[0]);
174
175        for (String s : COMM) System.out.print(s + " "); System.out.println();
176
177        Process pro = java.lang.Runtime.getRuntime().exec(COMM);
178
179        R reader1 = new R(pro.getInputStream());
180        R reader2 = new R(pro.getErrorStream());
181
182        Thread t1 = new Thread(reader1);
183        Thread t2 = new Thread(reader2);
184
185        t1.setDaemon(true);
186        t2.setDaemon(true);
187
188        t1.start();
189        t2.start();
190
191        int i = 0;
192
193        System.out.println("Waiting to read the code.");
194
195        int counter = 0;
196        while (reader1.stillReading && reader2.stillReading)
197            if (i++ % 500000000 == 0)
198            {
199                System.out.print(".");
200                if (++counter == maxCount) break;
201            }
202    }
203
204    private static class R implements Runnable
205    {
206        private BufferedReader  br;
207        public  boolean         stillReading = true;
208
209        public R(InputStream is)
210        { this.br = new BufferedReader(new InputStreamReader(is)); }
211
212        public void run()
213        {
214            try
215            {
216                String  s       = null;
217                long    time    = System.currentTimeMillis();
218
219                while ((s = br.readLine()) != null)
220                {
221                    System.out.println(s);
222
223                    long current = System.currentTimeMillis();
224
225                    if ((time - current) > 4000) break;
226
227                    time = current;
228                }
229                stillReading = false;
230            }
231
232            catch (IOException ioe)
233                { System.out.println(EXCC.toString(ioe));  System.exit(0); }
234        }
235    }
236
237
238    // ********************************************************************************************
239    // ********************************************************************************************
240    // Retrieving Browser Information
241    // ********************************************************************************************
242    // ********************************************************************************************
243
244
245    /**
246     * You may use this method (if you so choose, for whatever reason) to retrieve the Remote
247     * Debugging Port API from Google.  This method will return a (very long) JSON-File, as a
248     * {@code java.lang.String}
249     * 
250     * @param port This should be the port that was used to start the server.  This is, by default,
251     * port {@code 9223}.
252     * 
253     * @return A very-long JSON-{@code String} containing the currently-exported API that your
254     * web-browser provides.
255     * 
256     * @throws IOException If there are problems sending this request, or retrieving the response,
257     * this throws.
258     */
259    public static String readAPIFromServer(Integer port) throws IOException
260    {
261        if (port == null) port = 9223;
262
263        String  urlStr  = "http://127.0.0.1:" + port + "/json/protocol";
264        URL     url     = new URL(urlStr);
265
266        System.out.println("Querying WS Server for JSON\n" + url);
267
268        // NOTE: This is a very large JSON File - more than 20,000 lines of text
269        return Scrape.scrapePage(url);
270    }
271
272    /**
273     * This is supposed to be the easy part.  There is a very long "Identifier String" that
274     * contains some connection information.
275     * 
276     * <BR /><BR /><B STYLE='color: red;'>NOTE:</B> There is really no need to use this method,
277     * as this "Connection Stuff" is all taken care of inside this class (and the class
278     * {@link WebSocketSender}).
279     * 
280     * <BR /><BR />If, for some reason, there is a need to sue the feature of having multiple
281     * browser connections, and it is deemed important to build your own instance of 
282     * {@link WebSocketSender} - <I>you will have to retrieve this {@code URL} information in order
283     * to construct a {@link WebSocketSender} instance.</I>
284     * 
285     * @param port The browser-port upon. which the {@code WebSocket} was connected
286     * 
287     * @return An instance of {@link Ret2}:
288     * 
289     * <BR /><BR /><UL CLASS=JDUL>
290     * <LI><B>{@code String}</B> (Browser {@code URL})
291     *      <BR /><BR />The complete {@code URL} to use when connecting to the browser over its
292     *      RDP Connection.
293     *      <BR /><BR />
294     *      </LI>
295     * <LI><B>{@code String}</B> (Browser {@code URL})
296     *      <BR /><BR />A "Connection Code" that the browser uses to identify instances of a 
297     *      Web-Socket Connection.
298     *      <BR /><BR />
299     *      </LI>
300     * </UL>
301     * 
302     * @throws IOException When connecting to the browser, this is always a possibility.
303     */
304    public static Ret2<String, String> readURLToUseFromServer(Integer port) throws IOException
305    {
306        if (port == null) port = 9223;
307
308        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
309        // Retrieve the Web-Socket Address from URL - Attemt #1
310        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
311
312        URL url = new URL("http://127.0.0.1:" + port + "/json/list");
313
314        if (! QUIET) sw.println(
315            '\n' +
316            BRED + "ATTEMPT #1:" + RESET + " (Works when there is already a page opened)\n" +
317            "Querying WebSocketServer for Listening Address Using:\n\t" +
318            BYELLOW + url.toString() + RESET
319        );
320
321        try
322        {
323            String          json    = Scrape.scrapePage(url);
324            StringReader    sr      = new StringReader(json);
325
326            if (! QUIET) sw.println("Attempt #1, Server Responded With:\n" + json);
327
328            if (json.length() > 10) // else it gave us an empty response
329            {
330                JsonArray    jArr    = Json.createReader(sr).readArray();
331                JsonObject   jObj    = null; // used in next-section
332
333                if ((jArr != null) && (jArr.size() > 0))
334                {        
335                    String res  = jArr.getJsonObject(0).getString("webSocketDebuggerUrl", null);
336                    String code = StringParse.fromLastFrontSlashPos(res);
337
338                    if (res != null) return new Ret2<>(res, code);
339                }
340            }
341        }
342
343        catch (Exception e)
344        {
345            sw.println(
346                BGREEN + "NOTE: FOR ATTEMPT #1, THIS IS SOMEWHAT EXPECTED" + RESET + '\n' +
347                BRED + "Exception Message Attempt #1:\n\t" + RESET +
348                e.getMessage() + '\n' +
349                "-------------------------------------------------------------------\n" +
350                EXCC.toString(e) +
351                "-------------------------------------------------------------------\n"
352            );
353        }
354
355
356        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
357        // Retrieve the Web-Socket Address from URL #2
358        // *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
359
360        url = new URL("http://127.0.0.1:" + port + "/json/version");
361
362        // Again, I'm not 100% what chrome is really doing here...
363        // url = new URL("http://127.0.0.1/json/version");
364
365        if (! QUIET) sw.println(
366            BRED + "ATTEMPT #2:" + RESET + " (Creates an entirely new connection)\n" +
367            "Querying WebSocketServer for Listening Address Using:\n\t" +
368            BYELLOW + url.toString() + RESET
369        );
370
371        try
372        {
373            String          json    = Scrape.scrapePage(url);
374            StringReader    sr      = new StringReader(json);
375            JsonObject      jObj    = Json.createReader(sr).readObject();
376
377            if (! QUIET)
378                sw.println("Attempt #2, Server Responded With:\n" + json);
379
380            if (jObj != null)
381            {
382                String res  = jObj.getString("webSocketDebuggerUrl", null);
383                String code = StringParse.fromLastFrontSlashPos(res);
384
385                sw.println(
386                    "res:  " + res + '\n' +
387                    "code: " + code
388                );
389
390                if ((res != null) && res.startsWith("ws://")) return new Ret2<>(res, code);
391            }
392        }
393
394        catch (Exception e)
395        {
396            sw.println(
397                BRED + "Exception Message Attempt #2:\n\t" + RESET +
398                e.getMessage() + '\n' +
399                "-------------------------------------------------------------------\n" +
400                EXCC.toString(e) +
401                "-------------------------------------------------------------------\n"
402            );
403        }
404    
405        // Failed
406        sw.println(BRED + "Unable to retrieve Chrome Web-Socket URL.  Sorry buddy." + RESET);
407        return null;
408    }
409
410    /**
411     * After an instance of Chrome has been started, this method should be used to create a 
412     * <B>{@code WebSocket}</B>-Connection to the browser.
413     * 
414     * <BR /><BR /><H2 CLASS=JDBanner>You must invoked this method before sending commands</H2>
415     * 
416     * <I CLASS=JDBanner>After starting a browser instance using the method 
417     * {@link #startBrowserExecutable(boolean, Integer, String, int)}, you will also have to open
418     * a Web-Socket Connection to the browser using this method.  Then (and only then), is it 
419     * possible to begin sending requets to the browser.</I>
420     * 
421     * @param port You should just elect to use {@code 9223}, that is what is recommended by all of
422     * the sources on that Internet that explain this browser-feature.  Chrome will be listening on
423     * whatever port you started the headless version with.
424     * 
425     * <BR /><BR /><B>NOTE:</B> You don't have to start the browser in headless mode.  Controlling
426     * a full-fleged, and opened, browser is allowed with Chrome.
427     * 
428     * @param quiet You may configure the verbosity-level for the
429     * <B>{@code Websocket}</B>-Connection to be different from the verbosity-level for this class
430     * (class {@code BRDPC}).
431     * 
432     * @param eventHandler It is advisable to register an event-handler.  This parameter may be
433     * passed null, and if it is, events will not be handled (they are then ignored).
434     * 
435     * @throws IOException Connecting to the browser may cause this exception.
436     * 
437     * @throws WebSocketException The {@code NeoVisionaries} package for {@code WebSockets} may
438     * throw this when attempting to build the connection.
439     */
440    public static boolean buildDefaultWebSocketConnection
441        (Integer port, boolean quiet, Consumer<BrowserEvent> eventHandler)
442        throws IOException, WebSocketException
443    {
444        Ret2<String, String> browserListenerURL = readURLToUseFromServer(port);
445
446        if ((browserListenerURL == null) || (browserListenerURL.a == null))
447        {
448            sw.println("Unable to retrieve a Web-Socket Connection.  No Request URL's");
449            return false;
450        }
451
452        BRDPC.defaultSender = new WebSocketSender(
453            browserListenerURL.a, quiet,
454            (eventHandler == null)
455                ? null
456                : (Object o) -> eventHandler.accept((BrowserEvent) o)
457        );
458
459        if (! QUIET) sw.println
460            ("Web Socket Connection Opened:\n" + BYELLOW + browserListenerURL.a + RESET);
461
462        return true;
463    }    
464
465
466    // ********************************************************************************************
467    // ********************************************************************************************
468    // Exception throw methdods used by all of the Library-Classes
469    // ********************************************************************************************
470    // ********************************************************************************************
471
472
473    /** Configuration to Override the null-check. */
474    public static final boolean ALLOW_NULLABLE_PARAMETERS = false;
475
476    /**
477     * In order to differentiate between <B>{@code 'optional'}</B> (and required) parameters and
478     * fields, quite an amount of effort was invested in checking that required variables are
479     * not allowed to be null.
480     * 
481     * <BR /><BR />This method ensures a consistent-looking error message whenever there is a
482     * null parameter passed that should not have been null.
483     */
484    static void throwNPE(String requiredPropertyParameterName)
485    {
486        if (ALLOW_NULLABLE_PARAMETERS) return;
487
488        throw new NullPointerException(
489            "The parameter '" + requiredPropertyParameterName + "' is not marked is optional. " +
490            "This means that null may not be passed to this parameter in this method.  However " +
491            "this parameter received a null value."
492        );
493    }
494
495    /**
496     * In addition to null checks, a lot of the Browser's API provides its own version of the
497     * Java {@code 'enum'} by listing valid-{@code String's} (as pre-defined lists) that an
498     * {@code String} parameter or field will accept.
499     * 
500     * <BR /><BR />If a programmer ever provides a {@code String} to a {@code String} parameter
501     * or field for which an enumerated-list of valid-values of the parameter or field has been
502     * provided, <I>but that value doesn't meet the enumerated-list's requiremets</I>, then an
503     * {@code IllegalArgumentException} will throw.
504     * 
505     * @param parameterName The name of the offending parameter.
506     * @param parameterValue The {@code String}-Value of the offending parameter.
507     * 
508     * @throws IllegalArgumentException This method shall this exception, everytime.
509     */
510    static void throwIAE(String parameterName, String parameterValue, String... enumStrArr)
511    {
512        // NOTE: This presumes that 'non-optional' parameters have already been null-checked
513        if (parameterValue == null) return;
514
515        if (StrCmpr.equalsNAND(parameterValue, enumStrArr))
516
517            throw new IllegalArgumentException(
518                "The parameter '" + parameterName + "' is a String parameter whose values are " +
519                "limited to the Enumerated Strings listed in this method's documentation page.  " +
520                "Unfortunately, this method received [" + parameterValue + "] as a String, " +
521                "which is not among the listed values for this String Parameter."
522            );
523    }
524
525    /**
526     * Also throws exception for enumerated strings
527     */
528    static void checkIAE
529        (String parameterName, String parameterValue, String enumName, String... enumStrArr)
530    {
531        // NOTE: This presumes that 'non-optional' parameters have already been null-checked
532        if (parameterValue == null) return;
533
534        if (StrCmpr.equalsNAND(parameterValue, enumStrArr))
535
536            throw new IllegalArgumentException(
537                "The parameter '" + parameterName + "' is a String parameter whose values are " +
538                "limited to the Enumerated Strings listed in String-Array[] '" + enumName + "'.  " +
539                "Unfortunately, this method received [" + parameterValue + "] as a String, " +
540                "and that is not among the listed values for the String Parameter."
541            );
542    }
543}