From 9655efe8742dc2aa2292f6dbe29d2f038bca01ea Mon Sep 17 00:00:00 2001 From: 3timeslazy Date: Wed, 10 Apr 2019 13:20:20 +0300 Subject: [PATCH] sync with MontFerret/ferret --- .codecov.yml | 21 + .goreleaser.yml | 2 - .travis.yml | 26 +- CHANGELOG.md | 120 + Gopkg.lock | 311 -- Gopkg.toml | 50 - Makefile | 26 +- README.md | 203 +- cli/autocompleter.go | 50 + cli/browser/browser.go | 11 +- cli/browser/flags.go | 17 +- cli/browser/helpers.go | 19 +- cli/browser/launcher.go | 10 +- cli/browser/options.go | 6 + cli/exec.go | 9 +- cli/options.go | 51 +- cli/repl.go | 35 +- e2e/main.go | 76 +- e2e/pages/bootstrap/overview.html | 565 ---- e2e/pages/dynamic/components/app.js | 34 + e2e/pages/dynamic/components/layout.js | 27 + .../components/pages/events/appearable.js | 71 + .../components/pages/events/clickable.js | 56 + .../components/pages/events/hoverable.js | 47 + .../dynamic/components/pages/events/index.js | 79 + .../dynamic/components/pages/forms/index.js | 124 + e2e/pages/dynamic/components/pages/index.js | 12 + e2e/pages/dynamic/index.css | 4 + e2e/pages/dynamic/index.html | 21 + e2e/pages/dynamic/index.js | 9 + e2e/pages/dynamic/utils/qs.js | 82 + e2e/pages/dynamic/utils/random.js | 13 + .../{bootstrap => static}/assets/analytics.js | 0 .../assets/bootstrap.min.css | 0 .../assets/bootstrap.min.js | 0 .../{bootstrap => static}/assets/carbon.js | 0 .../{bootstrap => static}/assets/docs.min.css | 0 .../{bootstrap => static}/assets/docs.min.js | 0 .../assets/docsearch.min.css | 0 .../assets/docsearch.min.js | 0 .../assets/jquery-3.3.1.slim.min.js | 0 .../assets/popper.min.js | 0 e2e/pages/{bootstrap => static}/grid.html | 0 e2e/pages/{bootstrap => static}/media.html | 0 e2e/pages/static/overview.html | 1128 +++++++ e2e/pages/static/simple.html | 10 + .../{bootstrap => static}/utilities.html | 0 e2e/pages/static/value.html | 47 + e2e/runner/lib.go | 3 +- e2e/runner/runner.go | 99 +- e2e/server/server.go | 14 + e2e/tests/doc_cookie_del_d.fql | 21 + e2e/tests/doc_cookie_get_d.fql | 10 + e2e/tests/doc_cookie_load_d.fql | 14 + e2e/tests/doc_cookie_set_d.fql | 14 + e2e/tests/doc_element_exists.fql | 10 + e2e/tests/doc_element_exists_d.fql | 10 + e2e/tests/doc_hover_d.fql | 11 + .../{inner_html.fql => doc_inner_html.fql} | 2 +- e2e/tests/doc_inner_html_1_arg.fql | 19 + e2e/tests/doc_inner_html_1_arg_d.fql | 29 + ...er_html_all.fql => doc_inner_html_all.fql} | 2 +- e2e/tests/doc_inner_html_all_d.fql | 12 + e2e/tests/doc_inner_html_d.fql | 10 + .../{inner_text.fql => doc_inner_text.fql} | 2 +- e2e/tests/doc_inner_text_1_arg.fql | 10 + e2e/tests/doc_inner_text_1_arg_d.fql | 17 + ...er_text_all.fql => doc_inner_text_all.fql} | 2 +- e2e/tests/doc_inner_text_all_d.fql | 16 + e2e/tests/doc_inner_text_d.fql | 10 + e2e/tests/doc_input_text_d.fql | 10 + e2e/tests/doc_select_multi_d.fql | 9 + e2e/tests/doc_select_single_d.fql | 9 + e2e/tests/doc_wait_attr_all_d.fql | 9 + e2e/tests/doc_wait_attr_d.fql | 18 + e2e/tests/doc_wait_class_all_d.fql | 9 + e2e/tests/doc_wait_class_d.fql | 14 + e2e/tests/doc_wait_element_d.fql | 13 + e2e/tests/doc_wait_no_attr_d.fql | 16 + e2e/tests/doc_wait_no_class_all_d.fql | 9 + e2e/tests/doc_wait_no_class_d.fql | 14 + e2e/tests/doc_wait_no_element_d.fql | 13 + e2e/tests/doc_wait_no_style_all_d.fql | 30 + e2e/tests/doc_wait_no_style_d.fql | 19 + e2e/tests/doc_wait_style_all_d.fql | 21 + e2e/tests/doc_wait_style_d.fql | 15 + e2e/tests/el_attrs.fql | 13 + e2e/tests/el_attrs_d.fql | 13 + e2e/tests/el_attrs_get.d.fql | 11 + e2e/tests/el_attrs_remove.d.fql | 17 + e2e/tests/el_attrs_set.d.fql | 17 + e2e/tests/el_attrs_set_bulk.d.fql | 14 + e2e/tests/el_element_exists.fql | 12 + e2e/tests/el_element_exists_d.fql | 12 + e2e/tests/el_hover_d.fql | 13 + e2e/tests/el_inner_html_1_arg.fql | 11 + e2e/tests/el_inner_html_1_arg_d.fql | 11 + e2e/tests/el_inner_text_1_arg.fql | 11 + e2e/tests/el_inner_text_1_arg_d.fql | 14 + e2e/tests/el_input_text_d.fql | 11 + e2e/tests/el_select_multi_d.fql | 10 + e2e/tests/el_select_single_d.fql | 10 + e2e/tests/el_style_get.d.fql | 11 + e2e/tests/el_style_remove.d.fql | 17 + e2e/tests/el_style_set.d.fql | 17 + e2e/tests/el_style_set_bulk.d.fql | 17 + e2e/tests/el_value.fql | 12 + e2e/tests/el_value_d.fql | 12 + e2e/tests/el_wait_attr_2_d.fql | 14 + e2e/tests/el_wait_attr_d.fql | 20 + e2e/tests/el_wait_class_d.fql | 20 + e2e/tests/el_wait_no_attr_d.fql | 20 + e2e/tests/el_wait_no_class_d.fql | 20 + e2e/tests/el_wait_no_style_d.fql | 18 + e2e/tests/el_wait_style_d.fql | 18 + e2e/tests/inner_html_dynamic.fql | 10 - e2e/tests/inner_text_dynamic.fql | 9 - e2e/tests/{page-load.fql => page_load.fql} | 2 +- e2e/tests/page_load_d.fql | 4 + e2e/tests/page_load_params_d.fql | 6 + examples/cookies.go | 28 + examples/crawler.fql | 4 +- examples/embedded.go | 18 +- examples/extensible.go | 13 +- examples/navigate_forward_2.fql | 4 +- examples/pagination.fql | 24 +- examples/pagination_uncontrolled.fql | 36 + examples/wait_class.fql | 1 - go.mod | 76 + go.sum | 254 ++ main.go | 26 +- pkg/compiler/compiler.go | 24 +- pkg/compiler/compiler_collect_test.go | 45 + pkg/compiler/compiler_filter_test.go | 56 + pkg/compiler/compiler_for_test.go | 148 - pkg/compiler/compiler_let_test.go | 11 + pkg/compiler/compiler_limit_test.go | 96 + pkg/compiler/compiler_precedence_test.go | 65 + pkg/compiler/compiler_sort_test.go | 212 ++ pkg/compiler/compiler_str_test.go | 34 + pkg/compiler/compiler_test.go | 11 + pkg/compiler/result.go | 4 - pkg/compiler/scope.go | 16 +- pkg/compiler/visitor.go | 566 ++-- pkg/drivers/cdp/document.go | 1126 +++++++ pkg/{html/dynamic => drivers/cdp}/driver.go | 83 +- pkg/{html/dynamic => drivers/cdp}/element.go | 679 ++-- .../dynamic => drivers/cdp}/eval/eval.go | 9 +- pkg/drivers/cdp/eval/param.go | 36 + .../dynamic => drivers/cdp}/events/broker.go | 35 +- .../cdp}/events/broker_test.go | 15 +- .../cdp}/events/dispatch.go | 4 +- pkg/drivers/cdp/events/wait.go | 113 + pkg/drivers/cdp/helpers.go | 485 +++ pkg/drivers/cdp/options.go | 59 + pkg/drivers/cdp/templates/attributes.go | 14 + pkg/drivers/cdp/templates/helpers.go | 13 + pkg/drivers/cdp/templates/styles.go | 17 + pkg/drivers/cdp/templates/wait_by_selector.go | 36 + .../cdp/templates/wait_by_selector_all.go | 43 + pkg/{html => drivers}/common/attrs.go | 0 pkg/drivers/common/errors.go | 8 + pkg/drivers/common/getter.go | 141 + pkg/drivers/common/iterator.go | 37 + pkg/{html => drivers}/common/lazy.go | 19 +- pkg/drivers/common/path.go | 17 + pkg/drivers/common/setter.go | 130 + pkg/drivers/common/styles.go | 103 + pkg/drivers/common/styles_test.go | 102 + pkg/{html => drivers}/common/types.go | 0 pkg/{html => drivers}/common/ua.go | 5 +- pkg/drivers/cookie.go | 195 ++ pkg/drivers/driver.go | 84 + pkg/drivers/header.go | 135 + pkg/drivers/helpers.go | 9 + pkg/drivers/http/document.go | 292 ++ .../static => drivers/http}/document_test.go | 6 +- pkg/{html/static => drivers/http}/driver.go | 84 +- pkg/drivers/http/element.go | 481 +++ .../static => drivers/http}/element_test.go | 34 +- pkg/{html/static => drivers/http}/options.go | 18 +- pkg/drivers/options.go | 15 + pkg/drivers/pdf.go | 39 + pkg/drivers/screenshot.go | 47 + pkg/drivers/type.go | 42 + pkg/drivers/value.go | 172 + pkg/html/driver.go | 92 - pkg/html/dynamic/document.go | 819 ----- pkg/html/dynamic/events/wait.go | 84 - pkg/html/dynamic/helpers.go | 230 -- pkg/html/dynamic/options.go | 22 - pkg/html/static/document.go | 56 - pkg/html/static/element.go | 288 -- pkg/parser/antlr/FqlLexer.g4 | 6 +- pkg/parser/antlr/FqlLexer.tokens | 28 +- pkg/parser/antlr/FqlParser.g4 | 65 +- pkg/parser/fql/FqlLexer.interp | 20 +- pkg/parser/fql/FqlLexer.tokens | 28 +- pkg/parser/fql/FqlParser.interp | 28 +- pkg/parser/fql/FqlParser.tokens | 28 +- pkg/parser/fql/fql_lexer.go | 46 +- pkg/parser/fql/fql_parser.go | 2770 ++++++++++------- pkg/parser/fql/fqlparser_base_listener.go | 74 +- pkg/parser/fql/fqlparser_base_visitor.go | 40 +- pkg/parser/fql/fqlparser_listener.go | 74 +- pkg/parser/fql/fqlparser_visitor.go | 38 +- pkg/parser/parser.go | 2 +- pkg/runtime/collections/collection.go | 50 +- pkg/runtime/collections/data_set.go | 108 - pkg/runtime/collections/errors.go | 18 - pkg/runtime/collections/filter.go | 71 +- pkg/runtime/collections/filter_test.go | 103 +- pkg/runtime/collections/html.go | 40 - pkg/runtime/collections/indexed.go | 48 +- pkg/runtime/collections/indexed_test.go | 73 +- pkg/runtime/collections/iterator.go | 22 +- pkg/runtime/collections/keyed.go | 49 +- pkg/runtime/collections/keyed_test.go | 47 +- pkg/runtime/collections/limit.go | 76 +- pkg/runtime/collections/limit_test.go | 77 +- pkg/runtime/collections/map.go | 47 +- pkg/runtime/collections/map_test.go | 45 +- pkg/runtime/collections/noop.go | 13 - pkg/runtime/collections/slice.go | 39 +- pkg/runtime/collections/slice_test.go | 61 +- pkg/runtime/collections/sort.go | 70 +- pkg/runtime/collections/sort_test.go | 112 +- pkg/runtime/collections/tap.go | 43 + pkg/runtime/collections/tap_test.go | 128 + pkg/runtime/collections/unique.go | 63 +- pkg/runtime/collections/unique_test.go | 15 +- pkg/runtime/core/errors.go | 4 +- pkg/runtime/core/errors_test.go | 11 +- pkg/runtime/core/helpers.go | 21 +- pkg/runtime/core/scope.go | 133 +- pkg/runtime/core/scope_test.go | 245 +- pkg/runtime/core/type.go | 83 + pkg/runtime/core/type_test.go | 109 + pkg/runtime/core/value.go | 108 +- pkg/runtime/env/env.go | 31 - pkg/runtime/expressions/block.go | 73 +- pkg/runtime/expressions/block_test.go | 158 + pkg/runtime/expressions/body.go | 53 + pkg/runtime/expressions/body_test.go | 147 + pkg/runtime/expressions/clauses/collect.go | 56 +- .../expressions/clauses/collect_iterator.go | 214 +- pkg/runtime/expressions/clauses/filter.go | 42 +- pkg/runtime/expressions/clauses/limit.go | 56 +- pkg/runtime/expressions/clauses/sort.go | 46 +- pkg/runtime/expressions/condition.go | 4 +- pkg/runtime/expressions/data_source.go | 82 +- pkg/runtime/expressions/data_source_test.go | 163 + pkg/runtime/expressions/for.go | 164 +- pkg/runtime/expressions/func_call.go | 39 +- pkg/runtime/expressions/func_call_test.go | 89 + pkg/runtime/expressions/literals/object.go | 6 +- pkg/runtime/expressions/member.go | 8 +- pkg/runtime/expressions/operators/array.go | 4 +- pkg/runtime/expressions/operators/equality.go | 1 + pkg/runtime/expressions/operators/in.go | 4 +- pkg/runtime/expressions/operators/logical.go | 6 +- pkg/runtime/expressions/operators/math.go | 9 +- pkg/runtime/expressions/operators/operator.go | 78 +- pkg/runtime/expressions/operators/range.go | 14 +- pkg/runtime/expressions/operators/unary.go | 1 + pkg/runtime/expressions/param_test.go | 65 + pkg/runtime/expressions/return.go | 18 +- pkg/runtime/expressions/return_test.go | 87 + pkg/runtime/expressions/variable.go | 7 +- pkg/runtime/options.go | 63 +- pkg/runtime/program.go | 51 +- pkg/runtime/program_test.go | 54 + pkg/runtime/values/array.go | 100 +- pkg/runtime/values/array_test.go | 5 +- pkg/runtime/values/binary.go | 56 +- pkg/runtime/values/boolean.go | 25 +- pkg/runtime/values/boolean_test.go | 7 +- pkg/runtime/values/date_time.go | 26 +- pkg/runtime/values/float.go | 26 +- pkg/runtime/values/helpers.go | 360 ++- pkg/runtime/values/helpers_test.go | 397 +++ pkg/runtime/values/html.go | 49 - pkg/runtime/values/int.go | 95 +- pkg/runtime/values/none.go | 12 +- pkg/runtime/values/object.go | 78 +- pkg/runtime/values/object_test.go | 7 +- pkg/runtime/values/string.go | 31 +- pkg/runtime/values/string_test.go | 8 + pkg/runtime/values/types/helpers.go | 42 + pkg/runtime/values/types/helpers_test.go | 176 ++ pkg/runtime/values/types/types.go | 15 + pkg/runtime/values/types/types_test.go | 84 + pkg/stdlib/arrays/append.go | 5 +- pkg/stdlib/arrays/append_test.go | 6 +- pkg/stdlib/arrays/first.go | 3 +- pkg/stdlib/arrays/flatten.go | 7 +- pkg/stdlib/arrays/intersection.go | 3 +- pkg/stdlib/arrays/last.go | 3 +- pkg/stdlib/arrays/lib.go | 24 +- pkg/stdlib/arrays/minus.go | 3 +- pkg/stdlib/arrays/nth.go | 5 +- pkg/stdlib/arrays/pop.go | 3 +- pkg/stdlib/arrays/position.go | 5 +- pkg/stdlib/arrays/push.go | 7 +- pkg/stdlib/arrays/remove_nth.go | 5 +- pkg/stdlib/arrays/remove_value.go | 5 +- pkg/stdlib/arrays/remove_values.go | 5 +- pkg/stdlib/arrays/reverse.go | 35 - pkg/stdlib/arrays/shift.go | 3 +- pkg/stdlib/arrays/slice.go | 7 +- pkg/stdlib/arrays/sorted.go | 23 +- pkg/stdlib/arrays/sorted_unique.go | 32 +- pkg/stdlib/arrays/union.go | 5 +- pkg/stdlib/arrays/union_distinct.go | 6 +- pkg/stdlib/arrays/unique.go | 15 +- pkg/stdlib/arrays/unshift.go | 5 +- pkg/stdlib/collections/length.go | 25 +- pkg/stdlib/collections/lib.go | 3 +- pkg/stdlib/collections/reverse.go | 51 + .../{arrays => collections}/reverse_test.go | 31 +- pkg/stdlib/datetime/add_subtract.go | 101 + pkg/stdlib/datetime/add_subtract_test.go | 294 ++ pkg/stdlib/datetime/compare.go | 69 + pkg/stdlib/datetime/compare_test.go | 107 + pkg/stdlib/datetime/date.go | 34 + pkg/stdlib/datetime/date_test.go | 55 + pkg/stdlib/datetime/day.go | 28 + pkg/stdlib/datetime/day_test.go | 49 + pkg/stdlib/datetime/dayofweek.go | 28 + pkg/stdlib/datetime/dayofweek_test.go | 49 + pkg/stdlib/datetime/dayofyear.go | 29 + pkg/stdlib/datetime/dayofyear_test.go | 57 + pkg/stdlib/datetime/daysinmonth.go | 50 + pkg/stdlib/datetime/daysinmonth_test.go | 68 + pkg/stdlib/datetime/diff.go | 78 + pkg/stdlib/datetime/diff_test.go | 178 ++ pkg/stdlib/datetime/format.go | 34 + pkg/stdlib/datetime/format_test.go | 115 + pkg/stdlib/datetime/helpers_test.go | 71 + pkg/stdlib/datetime/hour.go | 28 + pkg/stdlib/datetime/hour_test.go | 49 + pkg/stdlib/datetime/leapyear.go | 32 + pkg/stdlib/datetime/leapyear_test.go | 49 + pkg/stdlib/datetime/lib.go | 26 + pkg/stdlib/datetime/millisecond.go | 28 + pkg/stdlib/datetime/millisecond_test.go | 60 + pkg/stdlib/datetime/minute.go | 28 + pkg/stdlib/datetime/minute_test.go | 49 + pkg/stdlib/datetime/month.go | 28 + pkg/stdlib/datetime/month_test.go | 49 + pkg/stdlib/datetime/now.go | 20 + pkg/stdlib/datetime/now_test.go | 27 + pkg/stdlib/datetime/quarter.go | 39 + pkg/stdlib/datetime/quarter_test.go | 50 + pkg/stdlib/datetime/second.go | 28 + pkg/stdlib/datetime/second_test.go | 49 + pkg/stdlib/datetime/unit.go | 113 + pkg/stdlib/datetime/year.go | 28 + pkg/stdlib/datetime/year_test.go | 49 + pkg/stdlib/html/attr_get.go | 41 + pkg/stdlib/html/attr_remove.go | 41 + pkg/stdlib/html/attr_set.go | 50 + pkg/stdlib/html/blob.go | 402 --- pkg/stdlib/html/click.go | 30 +- pkg/stdlib/html/click_all.go | 14 +- pkg/stdlib/html/cookie_del.go | 66 + pkg/stdlib/html/cookie_get.go | 58 + pkg/stdlib/html/cookie_set.go | 42 + pkg/stdlib/html/document.go | 301 +- pkg/stdlib/html/document_parse.go | 38 - pkg/stdlib/html/download.go | 45 + pkg/stdlib/html/element.go | 14 +- pkg/stdlib/html/element_exists.go | 22 + pkg/stdlib/html/elements.go | 6 +- pkg/stdlib/html/elements_count.go | 6 +- pkg/stdlib/html/hover.go | 72 + pkg/stdlib/html/inner_html.go | 27 +- pkg/stdlib/html/inner_html_all.go | 17 +- pkg/stdlib/html/inner_text.go | 20 +- pkg/stdlib/html/inner_text_all.go | 17 +- pkg/stdlib/html/input.go | 61 +- pkg/stdlib/html/lib.go | 145 +- pkg/stdlib/html/mouse_xy.go | 56 + pkg/stdlib/html/navigate.go | 22 +- pkg/stdlib/html/navigate_back.go | 22 +- pkg/stdlib/html/navigate_forward.go | 22 +- pkg/stdlib/html/pagination.go | 107 + pkg/stdlib/html/pdf.go | 302 ++ pkg/stdlib/html/screenshot.go | 165 + pkg/stdlib/html/scroll_bottom.go | 33 + pkg/stdlib/html/scroll_element.go | 52 + pkg/stdlib/html/scroll_top.go | 33 + pkg/stdlib/html/scroll_xy.go | 56 + pkg/stdlib/html/select.go | 62 + pkg/stdlib/html/style_get.go | 44 + pkg/stdlib/html/style_remove.go | 41 + pkg/stdlib/html/style_set.go | 44 + pkg/stdlib/html/wait_attr.go | 100 + pkg/stdlib/html/wait_attr_all.go | 77 + pkg/stdlib/html/wait_class.go | 86 +- pkg/stdlib/html/wait_class_all.go | 40 +- pkg/stdlib/html/wait_element.go | 45 +- pkg/stdlib/html/wait_navigation.go | 22 +- pkg/stdlib/html/wait_style.go | 100 + pkg/stdlib/html/wait_style_all.go | 65 + pkg/stdlib/lib.go | 5 + pkg/stdlib/math/abs.go | 3 +- pkg/stdlib/math/acos.go | 3 +- pkg/stdlib/math/asin.go | 3 +- pkg/stdlib/math/atan.go | 3 +- pkg/stdlib/math/atan2.go | 5 +- pkg/stdlib/math/average.go | 5 +- pkg/stdlib/math/ceil.go | 3 +- pkg/stdlib/math/cos.go | 3 +- pkg/stdlib/math/degrees.go | 3 +- pkg/stdlib/math/exp.go | 3 +- pkg/stdlib/math/exp2.go | 3 +- pkg/stdlib/math/floor.go | 3 +- pkg/stdlib/math/lib.go | 6 +- pkg/stdlib/math/log.go | 3 +- pkg/stdlib/math/log10.go | 3 +- pkg/stdlib/math/log2.go | 3 +- pkg/stdlib/math/max.go | 5 +- pkg/stdlib/math/mean.go | 6 +- pkg/stdlib/math/median.go | 14 +- pkg/stdlib/math/min.go | 5 +- pkg/stdlib/math/percentile.go | 18 +- pkg/stdlib/math/pow.go | 5 +- pkg/stdlib/math/radians.go | 3 +- pkg/stdlib/math/rand.go | 33 +- pkg/stdlib/math/range.go | 7 +- pkg/stdlib/math/round.go | 3 +- pkg/stdlib/math/sin.go | 3 +- pkg/stdlib/math/sqrt.go | 3 +- pkg/stdlib/math/stddev_population.go | 9 +- pkg/stdlib/math/stddev_sample.go | 9 +- pkg/stdlib/math/sum.go | 5 +- pkg/stdlib/math/tan.go | 3 +- pkg/stdlib/math/variance.go | 12 +- pkg/stdlib/math/variance_population.go | 5 +- pkg/stdlib/math/variance_sample.go | 5 +- pkg/stdlib/objects/has.go | 5 +- pkg/stdlib/objects/{keep.go => keep_keys.go} | 18 +- .../{keep_test.go => keep_keys_test.go} | 68 +- pkg/stdlib/objects/keys.go | 15 +- pkg/stdlib/objects/keys_test.go | 5 +- pkg/stdlib/objects/lib.go | 2 +- pkg/stdlib/objects/merge.go | 13 +- pkg/stdlib/objects/merge_recursive.go | 5 +- pkg/stdlib/objects/merge_recursive_test.go | 2 - pkg/stdlib/objects/values.go | 16 +- pkg/stdlib/objects/zip.go | 18 +- pkg/stdlib/strings/concat.go | 11 +- pkg/stdlib/strings/contains.go | 7 +- pkg/stdlib/strings/decode.go | 8 +- pkg/stdlib/strings/encode.go | 2 +- pkg/stdlib/strings/find.go | 13 +- pkg/stdlib/strings/fmt.go | 87 + pkg/stdlib/strings/fmt_test.go | 151 + pkg/stdlib/strings/json_test.go | 19 +- pkg/stdlib/strings/lib.go | 2 +- pkg/stdlib/strings/like.go | 12 +- pkg/stdlib/strings/random.go | 3 +- pkg/stdlib/strings/regex.go | 5 +- pkg/stdlib/strings/reverse.go | 30 - pkg/stdlib/strings/reverse_test.go | 29 - pkg/stdlib/strings/split.go | 3 +- pkg/stdlib/strings/substitute.go | 3 +- pkg/stdlib/strings/substr.go | 13 +- pkg/stdlib/strings/substr_test.go | 11 +- pkg/stdlib/types/is_array.go | 3 +- pkg/stdlib/types/is_binary.go | 3 +- pkg/stdlib/types/is_boolean.go | 3 +- pkg/stdlib/types/is_date_time.go | 3 +- pkg/stdlib/types/is_float.go | 3 +- pkg/stdlib/types/is_html_document.go | 3 +- pkg/stdlib/types/is_html_element.go | 3 +- pkg/stdlib/types/is_int.go | 3 +- pkg/stdlib/types/is_nan.go | 3 +- pkg/stdlib/types/is_none.go | 3 +- pkg/stdlib/types/is_object.go | 3 +- pkg/stdlib/types/is_string.go | 3 +- pkg/stdlib/types/to_array.go | 6 +- pkg/stdlib/types/to_boolean.go | 13 +- pkg/stdlib/types/to_float.go | 15 +- pkg/stdlib/types/to_int.go | 15 +- pkg/stdlib/utils/lib.go | 4 +- pkg/stdlib/utils/log.go | 4 +- pkg/stdlib/utils/wait.go | 12 +- 489 files changed, 20577 insertions(+), 7701 deletions(-) create mode 100644 .codecov.yml delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 cli/autocompleter.go delete mode 100644 e2e/pages/bootstrap/overview.html create mode 100644 e2e/pages/dynamic/components/app.js create mode 100644 e2e/pages/dynamic/components/layout.js create mode 100644 e2e/pages/dynamic/components/pages/events/appearable.js create mode 100644 e2e/pages/dynamic/components/pages/events/clickable.js create mode 100644 e2e/pages/dynamic/components/pages/events/hoverable.js create mode 100644 e2e/pages/dynamic/components/pages/events/index.js create mode 100644 e2e/pages/dynamic/components/pages/forms/index.js create mode 100644 e2e/pages/dynamic/components/pages/index.js create mode 100644 e2e/pages/dynamic/index.css create mode 100644 e2e/pages/dynamic/index.html create mode 100644 e2e/pages/dynamic/index.js create mode 100644 e2e/pages/dynamic/utils/qs.js create mode 100644 e2e/pages/dynamic/utils/random.js rename e2e/pages/{bootstrap => static}/assets/analytics.js (100%) rename e2e/pages/{bootstrap => static}/assets/bootstrap.min.css (100%) rename e2e/pages/{bootstrap => static}/assets/bootstrap.min.js (100%) rename e2e/pages/{bootstrap => static}/assets/carbon.js (100%) rename e2e/pages/{bootstrap => static}/assets/docs.min.css (100%) rename e2e/pages/{bootstrap => static}/assets/docs.min.js (100%) rename e2e/pages/{bootstrap => static}/assets/docsearch.min.css (100%) rename e2e/pages/{bootstrap => static}/assets/docsearch.min.js (100%) rename e2e/pages/{bootstrap => static}/assets/jquery-3.3.1.slim.min.js (100%) rename e2e/pages/{bootstrap => static}/assets/popper.min.js (100%) rename e2e/pages/{bootstrap => static}/grid.html (100%) rename e2e/pages/{bootstrap => static}/media.html (100%) create mode 100644 e2e/pages/static/overview.html create mode 100644 e2e/pages/static/simple.html rename e2e/pages/{bootstrap => static}/utilities.html (100%) create mode 100644 e2e/pages/static/value.html create mode 100644 e2e/tests/doc_cookie_del_d.fql create mode 100644 e2e/tests/doc_cookie_get_d.fql create mode 100644 e2e/tests/doc_cookie_load_d.fql create mode 100644 e2e/tests/doc_cookie_set_d.fql create mode 100644 e2e/tests/doc_element_exists.fql create mode 100644 e2e/tests/doc_element_exists_d.fql create mode 100644 e2e/tests/doc_hover_d.fql rename e2e/tests/{inner_html.fql => doc_inner_html.fql} (89%) create mode 100644 e2e/tests/doc_inner_html_1_arg.fql create mode 100644 e2e/tests/doc_inner_html_1_arg_d.fql rename e2e/tests/{inner_html_all.fql => doc_inner_html_all.fql} (84%) create mode 100644 e2e/tests/doc_inner_html_all_d.fql create mode 100644 e2e/tests/doc_inner_html_d.fql rename e2e/tests/{inner_text.fql => doc_inner_text.fql} (87%) create mode 100644 e2e/tests/doc_inner_text_1_arg.fql create mode 100644 e2e/tests/doc_inner_text_1_arg_d.fql rename e2e/tests/{inner_text_all.fql => doc_inner_text_all.fql} (97%) create mode 100644 e2e/tests/doc_inner_text_all_d.fql create mode 100644 e2e/tests/doc_inner_text_d.fql create mode 100644 e2e/tests/doc_input_text_d.fql create mode 100644 e2e/tests/doc_select_multi_d.fql create mode 100644 e2e/tests/doc_select_single_d.fql create mode 100644 e2e/tests/doc_wait_attr_all_d.fql create mode 100644 e2e/tests/doc_wait_attr_d.fql create mode 100644 e2e/tests/doc_wait_class_all_d.fql create mode 100644 e2e/tests/doc_wait_class_d.fql create mode 100644 e2e/tests/doc_wait_element_d.fql create mode 100644 e2e/tests/doc_wait_no_attr_d.fql create mode 100644 e2e/tests/doc_wait_no_class_all_d.fql create mode 100644 e2e/tests/doc_wait_no_class_d.fql create mode 100644 e2e/tests/doc_wait_no_element_d.fql create mode 100644 e2e/tests/doc_wait_no_style_all_d.fql create mode 100644 e2e/tests/doc_wait_no_style_d.fql create mode 100644 e2e/tests/doc_wait_style_all_d.fql create mode 100644 e2e/tests/doc_wait_style_d.fql create mode 100644 e2e/tests/el_attrs.fql create mode 100644 e2e/tests/el_attrs_d.fql create mode 100644 e2e/tests/el_attrs_get.d.fql create mode 100644 e2e/tests/el_attrs_remove.d.fql create mode 100644 e2e/tests/el_attrs_set.d.fql create mode 100644 e2e/tests/el_attrs_set_bulk.d.fql create mode 100644 e2e/tests/el_element_exists.fql create mode 100644 e2e/tests/el_element_exists_d.fql create mode 100644 e2e/tests/el_hover_d.fql create mode 100644 e2e/tests/el_inner_html_1_arg.fql create mode 100644 e2e/tests/el_inner_html_1_arg_d.fql create mode 100644 e2e/tests/el_inner_text_1_arg.fql create mode 100644 e2e/tests/el_inner_text_1_arg_d.fql create mode 100644 e2e/tests/el_input_text_d.fql create mode 100644 e2e/tests/el_select_multi_d.fql create mode 100644 e2e/tests/el_select_single_d.fql create mode 100644 e2e/tests/el_style_get.d.fql create mode 100644 e2e/tests/el_style_remove.d.fql create mode 100644 e2e/tests/el_style_set.d.fql create mode 100644 e2e/tests/el_style_set_bulk.d.fql create mode 100644 e2e/tests/el_value.fql create mode 100644 e2e/tests/el_value_d.fql create mode 100644 e2e/tests/el_wait_attr_2_d.fql create mode 100644 e2e/tests/el_wait_attr_d.fql create mode 100644 e2e/tests/el_wait_class_d.fql create mode 100644 e2e/tests/el_wait_no_attr_d.fql create mode 100644 e2e/tests/el_wait_no_class_d.fql create mode 100644 e2e/tests/el_wait_no_style_d.fql create mode 100644 e2e/tests/el_wait_style_d.fql delete mode 100644 e2e/tests/inner_html_dynamic.fql delete mode 100644 e2e/tests/inner_text_dynamic.fql rename e2e/tests/{page-load.fql => page_load.fql} (52%) create mode 100644 e2e/tests/page_load_d.fql create mode 100644 e2e/tests/page_load_params_d.fql create mode 100644 examples/cookies.go create mode 100644 examples/pagination_uncontrolled.fql create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/compiler/compiler_limit_test.go create mode 100644 pkg/compiler/compiler_precedence_test.go create mode 100644 pkg/compiler/compiler_sort_test.go create mode 100644 pkg/compiler/compiler_test.go create mode 100644 pkg/drivers/cdp/document.go rename pkg/{html/dynamic => drivers/cdp}/driver.go (68%) rename pkg/{html/dynamic => drivers/cdp}/element.go (56%) rename pkg/{html/dynamic => drivers/cdp}/eval/eval.go (92%) create mode 100644 pkg/drivers/cdp/eval/param.go rename pkg/{html/dynamic => drivers/cdp}/events/broker.go (92%) rename pkg/{html/dynamic => drivers/cdp}/events/broker_test.go (92%) rename pkg/{html/dynamic => drivers/cdp}/events/dispatch.go (88%) create mode 100644 pkg/drivers/cdp/events/wait.go create mode 100644 pkg/drivers/cdp/helpers.go create mode 100644 pkg/drivers/cdp/options.go create mode 100644 pkg/drivers/cdp/templates/attributes.go create mode 100644 pkg/drivers/cdp/templates/helpers.go create mode 100644 pkg/drivers/cdp/templates/styles.go create mode 100644 pkg/drivers/cdp/templates/wait_by_selector.go create mode 100644 pkg/drivers/cdp/templates/wait_by_selector_all.go rename pkg/{html => drivers}/common/attrs.go (100%) create mode 100644 pkg/drivers/common/errors.go create mode 100644 pkg/drivers/common/getter.go create mode 100644 pkg/drivers/common/iterator.go rename pkg/{html => drivers}/common/lazy.go (75%) create mode 100644 pkg/drivers/common/path.go create mode 100644 pkg/drivers/common/setter.go create mode 100644 pkg/drivers/common/styles.go create mode 100644 pkg/drivers/common/styles_test.go rename pkg/{html => drivers}/common/types.go (100%) rename pkg/{html => drivers}/common/ua.go (72%) create mode 100644 pkg/drivers/cookie.go create mode 100644 pkg/drivers/driver.go create mode 100644 pkg/drivers/header.go create mode 100644 pkg/drivers/helpers.go create mode 100644 pkg/drivers/http/document.go rename pkg/{html/static => drivers/http}/document_test.go (99%) rename pkg/{html/static => drivers/http}/driver.go (58%) create mode 100644 pkg/drivers/http/element.go rename pkg/{html/static => drivers/http}/element_test.go (96%) rename pkg/{html/static => drivers/http}/options.go (77%) create mode 100644 pkg/drivers/options.go create mode 100644 pkg/drivers/pdf.go create mode 100644 pkg/drivers/screenshot.go create mode 100644 pkg/drivers/type.go create mode 100644 pkg/drivers/value.go delete mode 100644 pkg/html/driver.go delete mode 100644 pkg/html/dynamic/document.go delete mode 100644 pkg/html/dynamic/events/wait.go delete mode 100644 pkg/html/dynamic/helpers.go delete mode 100644 pkg/html/dynamic/options.go delete mode 100644 pkg/html/static/document.go delete mode 100644 pkg/html/static/element.go delete mode 100644 pkg/runtime/collections/data_set.go delete mode 100644 pkg/runtime/collections/errors.go delete mode 100644 pkg/runtime/collections/html.go delete mode 100644 pkg/runtime/collections/noop.go create mode 100644 pkg/runtime/collections/tap.go create mode 100644 pkg/runtime/collections/tap_test.go create mode 100644 pkg/runtime/core/type.go create mode 100644 pkg/runtime/core/type_test.go delete mode 100644 pkg/runtime/env/env.go create mode 100644 pkg/runtime/expressions/block_test.go create mode 100644 pkg/runtime/expressions/body.go create mode 100644 pkg/runtime/expressions/body_test.go create mode 100644 pkg/runtime/expressions/data_source_test.go create mode 100644 pkg/runtime/expressions/func_call_test.go create mode 100644 pkg/runtime/expressions/param_test.go create mode 100644 pkg/runtime/expressions/return_test.go create mode 100644 pkg/runtime/program_test.go create mode 100644 pkg/runtime/values/helpers_test.go delete mode 100644 pkg/runtime/values/html.go create mode 100644 pkg/runtime/values/types/helpers.go create mode 100644 pkg/runtime/values/types/helpers_test.go create mode 100644 pkg/runtime/values/types/types.go create mode 100644 pkg/runtime/values/types/types_test.go delete mode 100644 pkg/stdlib/arrays/reverse.go create mode 100644 pkg/stdlib/collections/reverse.go rename pkg/stdlib/{arrays => collections}/reverse_test.go (54%) create mode 100644 pkg/stdlib/datetime/add_subtract.go create mode 100644 pkg/stdlib/datetime/add_subtract_test.go create mode 100644 pkg/stdlib/datetime/compare.go create mode 100644 pkg/stdlib/datetime/compare_test.go create mode 100644 pkg/stdlib/datetime/date.go create mode 100644 pkg/stdlib/datetime/date_test.go create mode 100644 pkg/stdlib/datetime/day.go create mode 100644 pkg/stdlib/datetime/day_test.go create mode 100644 pkg/stdlib/datetime/dayofweek.go create mode 100644 pkg/stdlib/datetime/dayofweek_test.go create mode 100644 pkg/stdlib/datetime/dayofyear.go create mode 100644 pkg/stdlib/datetime/dayofyear_test.go create mode 100644 pkg/stdlib/datetime/daysinmonth.go create mode 100644 pkg/stdlib/datetime/daysinmonth_test.go create mode 100644 pkg/stdlib/datetime/diff.go create mode 100644 pkg/stdlib/datetime/diff_test.go create mode 100644 pkg/stdlib/datetime/format.go create mode 100644 pkg/stdlib/datetime/format_test.go create mode 100644 pkg/stdlib/datetime/helpers_test.go create mode 100644 pkg/stdlib/datetime/hour.go create mode 100644 pkg/stdlib/datetime/hour_test.go create mode 100644 pkg/stdlib/datetime/leapyear.go create mode 100644 pkg/stdlib/datetime/leapyear_test.go create mode 100644 pkg/stdlib/datetime/lib.go create mode 100644 pkg/stdlib/datetime/millisecond.go create mode 100644 pkg/stdlib/datetime/millisecond_test.go create mode 100644 pkg/stdlib/datetime/minute.go create mode 100644 pkg/stdlib/datetime/minute_test.go create mode 100644 pkg/stdlib/datetime/month.go create mode 100644 pkg/stdlib/datetime/month_test.go create mode 100644 pkg/stdlib/datetime/now.go create mode 100644 pkg/stdlib/datetime/now_test.go create mode 100644 pkg/stdlib/datetime/quarter.go create mode 100644 pkg/stdlib/datetime/quarter_test.go create mode 100644 pkg/stdlib/datetime/second.go create mode 100644 pkg/stdlib/datetime/second_test.go create mode 100644 pkg/stdlib/datetime/unit.go create mode 100644 pkg/stdlib/datetime/year.go create mode 100644 pkg/stdlib/datetime/year_test.go create mode 100644 pkg/stdlib/html/attr_get.go create mode 100644 pkg/stdlib/html/attr_remove.go create mode 100644 pkg/stdlib/html/attr_set.go delete mode 100644 pkg/stdlib/html/blob.go create mode 100644 pkg/stdlib/html/cookie_del.go create mode 100644 pkg/stdlib/html/cookie_get.go create mode 100644 pkg/stdlib/html/cookie_set.go delete mode 100644 pkg/stdlib/html/document_parse.go create mode 100644 pkg/stdlib/html/download.go create mode 100644 pkg/stdlib/html/element_exists.go create mode 100644 pkg/stdlib/html/hover.go create mode 100644 pkg/stdlib/html/mouse_xy.go create mode 100644 pkg/stdlib/html/pagination.go create mode 100644 pkg/stdlib/html/pdf.go create mode 100644 pkg/stdlib/html/screenshot.go create mode 100644 pkg/stdlib/html/scroll_bottom.go create mode 100644 pkg/stdlib/html/scroll_element.go create mode 100644 pkg/stdlib/html/scroll_top.go create mode 100644 pkg/stdlib/html/scroll_xy.go create mode 100644 pkg/stdlib/html/select.go create mode 100644 pkg/stdlib/html/style_get.go create mode 100644 pkg/stdlib/html/style_remove.go create mode 100644 pkg/stdlib/html/style_set.go create mode 100644 pkg/stdlib/html/wait_attr.go create mode 100644 pkg/stdlib/html/wait_attr_all.go create mode 100644 pkg/stdlib/html/wait_style.go create mode 100644 pkg/stdlib/html/wait_style_all.go rename pkg/stdlib/objects/{keep.go => keep_keys.go} (69%) rename pkg/stdlib/objects/{keep_test.go => keep_keys_test.go} (63%) create mode 100644 pkg/stdlib/strings/fmt.go create mode 100644 pkg/stdlib/strings/fmt_test.go delete mode 100644 pkg/stdlib/strings/reverse.go delete mode 100644 pkg/stdlib/strings/reverse_test.go diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..bab3d4d9 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,21 @@ +ignore: +# Ignore coverage for generated code. +- "pkg/parser" + +coverage: + range: 70..100 + round: nearest + precision: 1 + + status: + project: + default: + enabled: yes + threshold: 2% + patch: no + changes: no + +comment: + layout: "header, diff" + behavior: once + require_changes: yes \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 0b464024..89174716 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -12,9 +12,7 @@ builds: - darwin - windows goarch: - - 386 - amd64 - - arm - arm64 archive: diff --git a/.travis.yml b/.travis.yml index 69d04d1a..a69d1dc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,9 @@ os: - linux go: -- "1.10" -- "1.11" -- master +- "1.11.x" +- "1.12.x" +- stable addons: apt: @@ -16,19 +16,16 @@ addons: - oracle-java8-set-default chrome: stable -before_install: -- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +install: - go get -u github.com/mgechev/revive - sudo curl -o /usr/local/lib/antlr-4.7.1-complete.jar https://www.antlr.org/download/antlr-4.7.1-complete.jar - export CLASSPATH=".:/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" - mkdir $HOME/travis-bin -- echo -e '#!/bin/bash\njava -jar /usr/local/lib/antlr-4.7.1-complete.jar "$@"' > $HOME/travis-bin/antlr4 +- echo -e '#!/bin/bash\njava -jar /usr/local/lib/antlr-4.7.1-complete.jar "$@"' > $HOME/travis-bin/antlr - echo -e '#!/bin/bash\njava org.antlr.v4.gui.TestRig "$@"' > $HOME/travis-bin/grun - chmod +x $HOME/travis-bin/* - export PATH=$PATH:$HOME/travis-bin - -install: -- make install +- export GO111MODULE=on stages: - lint @@ -40,19 +37,23 @@ stages: jobs: include: - stage: lint + go: stable script: - make vet - make lint + - make fmt + - git diff + - if [[ $(git diff) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi - stage: compile + go: stable script: - make generate - make compile - stage: test script: - make cover - after_success: - - if [ $TRAVIS_BRANCH == "master" && $TRAVIS_PULL_REQUEST == "false" ]; then bash <(curl -s https://codecov.io/bash); fi - stage: e2e + go: stable before_script: - google-chrome-stable --headless --disable-gpu --remote-debugging-port=9222 --disable-setuid-sandbox --no-sandbox about:blank & script: @@ -60,5 +61,6 @@ jobs: after_script: - killall google-chrome-stable - stage: bench + go: stable script: - - make bench \ No newline at end of file + - make bench diff --git a/CHANGELOG.md b/CHANGELOG.md index f0a80a1f..62802958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,125 @@ ## Changelog +### 0.7.0 +#### Added +- Autocomplete to CLI [#219](https://github.com/MontFerret/ferret/pull/219). +- New mouse functions - ``MOUSE(x, y)`` and ``SCROLL(x, y)`` [#237](https://github.com/MontFerret/ferret/pull/237). +- ``WAIT_NO_ELEMENT``, ``WAIT_NO_CLASS`` and ``WAIT_NO_CLASS_ALL`` functions [#249](https://github.com/MontFerret/ferret/pull/249). +- Computed ``HTMLElement.style`` property [#255](https://github.com/MontFerret/ferret/pull/255). +- ``ATTR_GET``, ``ATTR_SET``, ``ATTR_REMOVE``, ``STYLE_GET``, ``STYLE_SET`` and ``STYLE_REMOVE`` functions [#255](https://github.com/MontFerret/ferret/pull/255). +- ``WAIT_STYLE``, ``WAIT_NO_STYLE``, ``WAIT_STYLE_ALL`` and ``WAIT_NO_STYLE_ALL`` functions [#256](https://github.com/MontFerret/ferret/pull/260). +- Cookies support. Now a document can be loaded with preset cookies. Also, HTMLDocument has ``.cookies`` property. +In order to manipulate with cookies, ``COOKIE_DEL``, ``COOKIE_SET`` AND ``COOKIE_GET`` functions were added [#242](https://github.com/MontFerret/ferret/pull/242). + +``` +LET doc = DOCUMENT(url, { + driver: "cdp", + cookies: [{ + name: "x-e2e", + value: "test" + }, { + name: "x-e2e-2", + value: "test2" + }] +}) +``` + +#### Changed +- Renamed ParseTYPEP to MustParseTYPE [#231](https://github.com/MontFerret/ferret/pull/231). +- Added context to all HTML object [#235](https://github.com/MontFerret/ferret/pull/235). + +#### Fixed +- Click events are not cancellable [#222](https://github.com/MontFerret/ferret/pull/222). +- Name collision [#223](https://github.com/MontFerret/ferret/pull/223). +- Invalid return in FQL Compiler constructor [#227](https://github.com/MontFerret/ferret/pull/227). +- Incorrect string length computation [#238](https://github.com/MontFerret/ferret/pull/238). +- Access to HTML object properties via dot notation [#239](https://github.com/MontFerret/ferret/pull/239). +- Graceful process termination [#240](https://github.com/MontFerret/ferret/pull/240). +- Browser launcher for macOS [#246](https://github.com/MontFerret/ferret/pull/246). + +#### Breaking changes +- New runtime type system [#232](https://github.com/MontFerret/ferret/pull/232). +- Moved and renamed ``collections.IterableCollection`` and ```collections.CollectionIterator``` interfaces. +Now they are in ``core`` package and called ``Iterable`` and ``Iterator`` [1af8b37](https://github.com/MontFerret/ferret/commit/f8e061cc8034fd4cfa4ce2a094276d50137a4b98). +- Renamed ``collections.Collection`` interface to ``collections.Measurable`` [1af8b37](https://github.com/MontFerret/ferret/commit/f8e061cc8034fd4cfa4ce2a094276d50137a4b98). +- Moved html interfaces from ``runtime/values`` package into ``drivers`` package [#234](https://github.com/MontFerret/ferret/pull/234). +- Changed drivers initialization. Replaced old ``drivers.WithDynamic`` and ``drivers.WithStatic`` methods with a new ``drivers.WithContext`` method with optional parameter ``drivers.AsDefault()`` [#234](https://github.com/MontFerret/ferret/pull/234). +- New document load params [#234](https://github.com/MontFerret/ferret/pull/234). +``` +LET doc = DOCUMENT(url, { + driver: "cdp" +}) +``` + + +### 0.6.0 +#### Added +- Added support for ```context.Done()``` to interrupt an execution [#201](https://github.com/MontFerret/ferret/pull/201). +- Added support for custom HTML drivers [#209](https://github.com/MontFerret/ferret/pull/209). +- Added support for dot notation access and assignments for custom types [#214](https://github.com/MontFerret/ferret/pull/214/commits/0ea36e511540e569ef53b8748301512b6d8a046b) +- Added ```ELEMENT_EXISTS(doc, selector) -> Boolean``` function [#210](https://github.com/MontFerret/ferret/pull/210). +``` +LET exists = ELEMENT_EXISTS(doc, ".nav") +``` +- Added ```PageLoadParams``` to ```DOCUMENT``` function [#214](https://github.com/MontFerret/ferret/pull/214/commits/3434323cd08ca3186e90cb5ab1faa26e28a28709). +``` +LET doc = DOCUMENT("https://www.google.com/", { + dynamic: true, + timeout: 10000 +}) +``` + +#### Fixed +- Math operators precedence [#202](https://github.com/MontFerret/ferret/pull/202). +- Memory leak in ```DOWNLOAD``` function [#213](https://github.com/MontFerret/ferret/pull/213). + +#### Breaking change +- **(Embedded)** Removed builtin drivers initialization in Program [#198](https://github.com/MontFerret/ferret/pull/198). +The initialization must be done via context manually. + +### 0.5.2 +#### Fixed +- Does not close browser tab when fails to load a page [#193](https://github.com/MontFerret/ferret/pull/193). +- ```HTMLElement.value``` does not return actual value [#195](https://github.com/MontFerret/ferret/pull/195) +- Compiles a query with duplicate variable in FOR statement [#196](https://github.com/MontFerret/ferret/pull/196) +- Default CDP address [#197](https://github.com/MontFerret/ferret/pull/197). + +### 0.5.1 +#### Fixed +- Unable to change a page load timeout [#186](https://github.com/MontFerret/ferret/pull/186). +- ``RETURN doc`` returns an empty string [#187](https://github.com/MontFerret/ferret/pull/187). +- Unable to pass an HTML Node without a selector to ``INNER_TEXT`` and ``INNER_HTML`` [#187](https://github.com/MontFerret/ferret/pull/187). +- ``doc.innerText`` returns an error [#187](https://github.com/MontFerret/ferret/pull/187). +- Panics when ``WAIT_CLASS`` does not receive all required arguments [#192](https://github.com/MontFerret/ferret/pull/192). + +### 0.5.0 +#### Added +- ``FMT`` function [#151](https://github.com/MontFerret/ferret/pull/151). +- DateTime functions [#152](https://github.com/MontFerret/ferret/pull/152), [#153](https://github.com/MontFerret/ferret/pull/153), [#154](https://github.com/MontFerret/ferret/pull/154), [#156](https://github.com/MontFerret/ferret/pull/156), [#157](https://github.com/MontFerret/ferret/pull/157), [#165](https://github.com/MontFerret/ferret/pull/165), [#175](https://github.com/MontFerret/ferret/pull/175), [#182](https://github.com/MontFerret/ferret/pull/182). +- ``PAGINATION`` function [#173](https://github.com/MontFerret/ferret/pull/173). +- ``SCROLL_TOP``, ``SCROLL_BOTTOM`` and ``SCROLL_ELEMENT`` functions [#174](https://github.com/MontFerret/ferret/pull/174). +- ``HOVER`` function [#178](https://github.com/MontFerret/ferret/pull/178). +- Panic recovery mechanism [#158](https://github.com/MontFerret/ferret/pull/158). + +#### Fixed +- Unable to define variables and make function calls before FILTER, SORT and etc statements [#148](https://github.com/MontFerret/ferret/pull/148). +- Unable to use params in LIMIT clause [#173](https://github.com/MontFerret/ferret/pull/173). +- ```RIGHT``` should return substr counting from right rather than left [#164](https://github.com/MontFerret/ferret/pull/164). +- ``INNER_HTML`` returns outer HTML instead for dynamic elements [#170](https://github.com/MontFerret/ferret/pull/170). +- ``INNER_TEXT`` returns HTML instead from dynamic elements [#170](https://github.com/MontFerret/ferret/pull/170). + +#### Breaking change: +- Name collision between ```math``` and ```utils``` packages in standard library. Renamed ```LOG``` to ```PRINT``` [#162](https://github.com/MontFerret/ferret/pull/162). + +### 0.4.0 +#### Added +- ``COLLECT`` keyword [#141](https://github.com/MontFerret/ferret/pull/141) +- ``VALUES`` function [#128](https://github.com/MontFerret/ferret/pull/128) +- ``MERGE_RECURSIVE`` function [#140](https://github.com/MontFerret/ferret/pull/140) + +#### Fixed +- Unable to use string literals as object properties [commit](https://github.com/MontFerret/ferret/commit/685c5872aaed42852ce32e7ab8b69b1a269185be) + ### 0.3.0 #### Added diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 4eaf26b9..00000000 --- a/Gopkg.lock +++ /dev/null @@ -1,311 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:a62f6ed230a8cd138a9efbe718e7d0b0294f139266f5f55cd942769a9aac8de2" - name = "github.com/PuerkitoBio/goquery" - packages = ["."] - pruneopts = "UT" - revision = "dc2ec5c7ca4d9aae063b79b9f581dd3ea6afd2b2" - version = "v1.4.1" - -[[projects]] - digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11" - name = "github.com/andybalholm/cascadia" - packages = ["."] - pruneopts = "UT" - revision = "901648c87902174f774fac311d7f176f8647bdaa" - version = "v1.0.0" - -[[projects]] - digest = "1:5e79f7a65816b2ba311439df21c9b24af91868cbb5d67489be6d94be2cccb286" - name = "github.com/antlr/antlr4" - packages = ["runtime/Go/antlr"] - pruneopts = "UT" - revision = "bdc05c87be2ad981744223df0fd745e8345baba9" - version = "4.7.1" - -[[projects]] - digest = "1:b95738a1e6ace058b5b8544303c0871fc01d224ef0d672f778f696265d0f2917" - name = "github.com/chzyer/readline" - packages = ["."] - pruneopts = "UT" - revision = "62c6fe6193755f722b8b8788aa7357be55a50ff1" - version = "v1.4" - -[[projects]] - branch = "master" - digest = "1:cc439e1d9d8cff3d575642f5401033b00f2b8d0cd9f859db45604701c990879a" - name = "github.com/corpix/uarand" - packages = ["."] - pruneopts = "UT" - revision = "2b8494104d86337cdd41d0a49cbed8e4583c0ab4" - -[[projects]] - digest = "1:ce579162ae1341f3e5ab30c0dce767f28b1eb6a81359aad01723f1ba6b4becdf" - name = "github.com/gofrs/uuid" - packages = ["."] - pruneopts = "UT" - revision = "370558f003bfe29580cd0f698d8640daccdcc45c" - version = "v3.1.1" - -[[projects]] - branch = "master" - digest = "1:f14d1b50e0075fb00177f12a96dd7addf93d1e2883c25befd17285b779549795" - name = "github.com/gopherjs/gopherjs" - packages = ["js"] - pruneopts = "UT" - revision = "0210a2f0f73c96103378b0b935f39868e5731809" - -[[projects]] - digest = "1:7b5c6e2eeaa9ae5907c391a91c132abfd5c9e8a784a341b5625e750c67e6825d" - name = "github.com/gorilla/websocket" - packages = ["."] - pruneopts = "UT" - revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" - version = "v1.4.0" - -[[projects]] - digest = "1:1e15f4e455f94aeaedfcf9c75b3e1c449b5acba1551c58446b4b45be507c707b" - name = "github.com/jtolds/gls" - packages = ["."] - pruneopts = "UT" - revision = "77f18212c9c7edc9bd6a33d383a7b545ce62f064" - version = "v4.2.1" - -[[projects]] - digest = "1:63801c64679bdeef1b63206f065d5a1f4c3a75ed66a3fbbd909bf98766542980" - name = "github.com/labstack/echo" - packages = ["."] - pruneopts = "UT" - revision = "1abaa3049251d17932e4313c2d6165073fd07fd8" - version = "v3.3.6" - -[[projects]] - digest = "1:faee5b9f53eb1ae4eb04708c040c8c4dd685ce46509671e57a08520a15c54368" - name = "github.com/labstack/gommon" - packages = [ - "color", - "log", - ] - pruneopts = "UT" - revision = "2a618302b929cc20862dda3aa6f02f64dbe740dd" - version = "v0.2.7" - -[[projects]] - digest = "1:7ffdd69928c5153fc351132aa80bbcc18a8f8122de1ba592cf42dccb65732361" - name = "github.com/mafredri/cdp" - packages = [ - ".", - "devtool", - "internal/errors", - "protocol", - "protocol/accessibility", - "protocol/animation", - "protocol/applicationcache", - "protocol/audits", - "protocol/browser", - "protocol/cachestorage", - "protocol/console", - "protocol/css", - "protocol/database", - "protocol/debugger", - "protocol/deviceorientation", - "protocol/dom", - "protocol/domdebugger", - "protocol/domsnapshot", - "protocol/domstorage", - "protocol/emulation", - "protocol/headlessexperimental", - "protocol/heapprofiler", - "protocol/indexeddb", - "protocol/input", - "protocol/inspector", - "protocol/internal", - "protocol/io", - "protocol/layertree", - "protocol/log", - "protocol/memory", - "protocol/network", - "protocol/overlay", - "protocol/page", - "protocol/performance", - "protocol/profiler", - "protocol/runtime", - "protocol/schema", - "protocol/security", - "protocol/serviceworker", - "protocol/storage", - "protocol/systeminfo", - "protocol/target", - "protocol/testing", - "protocol/tethering", - "protocol/tracing", - "rpcc", - "session", - ] - pruneopts = "UT" - revision = "75b0ecc5efcff27ac756a33ec71f0db75dc3d21c" - version = "v0.19.0" - -[[projects]] - digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" - name = "github.com/mattn/go-colorable" - packages = ["."] - pruneopts = "UT" - revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" - version = "v0.0.9" - -[[projects]] - digest = "1:0981502f9816113c9c8c4ac301583841855c8cf4da8c72f696b3ebedf6d0e4e5" - name = "github.com/mattn/go-isatty" - packages = ["."] - pruneopts = "UT" - revision = "6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c" - version = "v0.0.4" - -[[projects]] - digest = "1:c805e517269b0ba4c21ded5836019ed7d16953d4026cb7d00041d039c7906be9" - name = "github.com/natefinch/lumberjack" - packages = ["."] - pruneopts = "UT" - revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" - version = "v2.1" - -[[projects]] - digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" - name = "github.com/pkg/errors" - packages = ["."] - pruneopts = "UT" - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - digest = "1:5bc8f93f977b72a7a5264725c3bab275e69de0cc3e5c0dc1ee56feb564c33f03" - name = "github.com/rs/zerolog" - packages = [ - ".", - "internal/cbor", - "internal/json", - ] - pruneopts = "UT" - revision = "338f9bc14084d22cb8eeacd6492861f8449d715c" - version = "v1.9.1" - -[[projects]] - digest = "1:4ca145a665316d3c020a39c0741780fa3636b9152b824206796c4dce541f4a24" - name = "github.com/sethgrid/pester" - packages = ["."] - pruneopts = "UT" - revision = "03e26c9abbbf5accb8349790bf9f41bde09d72c3" - version = "1.0.0" - -[[projects]] - digest = "1:cc1c574c9cb5e99b123888c12b828e2d19224ab6c2244bda34647f230bf33243" - name = "github.com/smartystreets/assertions" - packages = [ - ".", - "internal/go-render/render", - "internal/oglematchers", - ] - pruneopts = "UT" - revision = "7678a5452ebea5b7090a6b163f844c133f523da2" - version = "1.8.3" - -[[projects]] - digest = "1:a3e081e593ee8e3b0a9af6a5dcac964c67a40c4f2034b5345b2ad78d05920728" - name = "github.com/smartystreets/goconvey" - packages = [ - "convey", - "convey/gotest", - "convey/reporting", - ] - pruneopts = "UT" - revision = "9e8dc3f972df6c8fcc0375ef492c24d0bb204857" - version = "1.6.3" - -[[projects]] - digest = "1:c468422f334a6b46a19448ad59aaffdfc0a36b08fdcc1c749a0b29b6453d7e59" - name = "github.com/valyala/bytebufferpool" - packages = ["."] - pruneopts = "UT" - revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" - version = "v1.0.0" - -[[projects]] - branch = "master" - digest = "1:268b8bce0064e8c057d7b913605459f9a26dcab864c0886a56d196540fbf003f" - name = "github.com/valyala/fasttemplate" - packages = ["."] - pruneopts = "UT" - revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" - -[[projects]] - branch = "master" - digest = "1:dedf20eb0d3e8d6aa8a4d3d2fae248222b688ed528201995e152cc497899123c" - name = "golang.org/x/crypto" - packages = [ - "acme", - "acme/autocert", - ] - pruneopts = "UT" - revision = "a92615f3c49003920a58dedcf32cf55022cefb8d" - -[[projects]] - branch = "master" - digest = "1:24641a15105cc4281b8a4a701ac4019194618102c327b1495485b3f19ffdd53e" - name = "golang.org/x/net" - packages = [ - "context", - "html", - "html/atom", - ] - pruneopts = "UT" - revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" - -[[projects]] - branch = "master" - digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239" - name = "golang.org/x/sync" - packages = ["errgroup"] - pruneopts = "UT" - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" - -[[projects]] - branch = "master" - digest = "1:0a40b0bdd57a93e741d8557465be3a2edeec408e9b6399586ad65bbe8e355796" - name = "golang.org/x/sys" - packages = ["unix"] - pruneopts = "UT" - revision = "fa43e7bc11baaae89f3f902b2b4d832b68234844" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/PuerkitoBio/goquery", - "github.com/antlr/antlr4/runtime/Go/antlr", - "github.com/chzyer/readline", - "github.com/corpix/uarand", - "github.com/gofrs/uuid", - "github.com/labstack/echo", - "github.com/mafredri/cdp", - "github.com/mafredri/cdp/devtool", - "github.com/mafredri/cdp/protocol/dom", - "github.com/mafredri/cdp/protocol/emulation", - "github.com/mafredri/cdp/protocol/input", - "github.com/mafredri/cdp/protocol/page", - "github.com/mafredri/cdp/protocol/runtime", - "github.com/mafredri/cdp/protocol/target", - "github.com/mafredri/cdp/rpcc", - "github.com/mafredri/cdp/session", - "github.com/natefinch/lumberjack", - "github.com/pkg/errors", - "github.com/rs/zerolog", - "github.com/sethgrid/pester", - "github.com/smartystreets/goconvey/convey", - "golang.org/x/net/html", - "golang.org/x/sync/errgroup", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 5ae873d6..00000000 --- a/Gopkg.toml +++ /dev/null @@ -1,50 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[prune] - go-tests = true - unused-packages = true - -[[constraint]] - name = "github.com/antlr/antlr4" - version = "4.7.1" - -[[constraint]] - name = "github.com/mafredri/cdp" - version = "0.19.0" - -[[constraint]] - name = "github.com/chzyer/readline" - version = "1.4" - -[[constraint]] - name = "github.com/PuerkitoBio/goquery" - version = "1.4.1" - -[[constraint]] - name = "github.com/gofrs/uuid" - version = "3.1.1" \ No newline at end of file diff --git a/Makefile b/Makefile index 76ffba3c..2af21539 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -.PHONY: build compile install test e2e doc fmt lint vet release +.PHONY: build compile test e2e doc fmt lint vet release export GOPATH VERSION ?= $(shell git describe --tags --always --dirty) +RELEASE_VERSION ?= $(version) DIR_BIN = ./bin DIR_PKG = ./pkg DIR_CLI = ./cli @@ -10,24 +11,22 @@ DIR_E2E = ./e2e default: build -build: install vet generate test compile +build: vet generate test compile compile: go build -v -o ${DIR_BIN}/ferret \ -ldflags "-X main.version=${VERSION}" \ ./main.go -install: - dep ensure - test: - go test -v -race ${DIR_PKG}/... + go test -race -v ${DIR_PKG}/... cover: - go test -race -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... + go test -race -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... && \ + curl -s https://codecov.io/bash | bash e2e: - go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages + go run ${DIR_E2E}/main.go --tests ${DIR_E2E}/tests --pages ${DIR_E2E}/pages --filter doc_cookie_set* bench: go test -run=XXX -bench=. ${DIR_PKG}/... @@ -53,4 +52,13 @@ vet: go vet ${DIR_CLI}/... ${DIR_PKG}/... release: - goreleaser \ No newline at end of file +ifeq ($(RELEASE_VERSION), ) + $(error "Release version is required (version=x)") +else ifeq ($(GITHUB_TOKEN), ) + $(error "GitHub token is required (GITHUB_TOKEN)") +else + rm -rf ./dist && \ + git tag -a v$(RELEASE_VERSION) -m "New $(RELEASE_VERSION) version" && \ + git push origin v$(RELEASE_VERSION) && \ + goreleaser +endif diff --git a/README.md b/README.md index 5071962f..b613f08d 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,38 @@ # Ferret

- Build Status - - - - Discord Chat - MIT License + + Go Report Status + + + Build Status + + + + + + Discord Chat + + + Ferret release + + + MIT License +

![ferret](https://raw.githubusercontent.com/MontFerret/ferret/master/assets/intro.jpg) ## What is it? -```ferret``` is a web scraping system aiming to simplify data extraction from the web for such things like UI testing, machine learning and analytics. -Having its own declarative language, ```ferret``` abstracts away technical details and complexity of the underlying technologies, helping to focus on the data itself. -It's extremely portable, extensible and fast. +```ferret``` is a web scraping system. It aims to simplify data extraction from the web for UI testing, machine learning, analytics and more. +```ferret``` allows users to focus on the data. It abstracts away the technical details and complexity of underlying technologies using its own declarative language. +It is extremely portable, extensible and fast. + +[Read the introductory blog post about Ferret here!](https://medium.com/@ziflex/say-hello-to-ferret-a-modern-web-scraping-tool-5c9cc85ba183) ## Show me some code The following example demonstrates the use of dynamic pages. -First of all, we load the main Google Search page, type search criteria into an input box and then click a search button. -The click action triggers a redirect, so we wait till its end. +We load the main Google Search page, type search criteria into an input box and then click a search button. +The click action triggers a redirect, so we wait until its end. Once the page gets loaded, we iterate over all elements in search results and assign the output to a variable. The final for loop filters out empty elements that might be because of inaccurate use of selectors. @@ -71,11 +85,10 @@ You can download latest binaries from [here](https://github.com/MontFerret/ferre ### Source code #### Production -* Go >=1.10 +* Go >=1.11 * Chrome or Docker #### Development -* GoDep * GNU Make * ANTLR4 >=4.7.1 @@ -90,8 +103,8 @@ In order to use all Ferret features, you will need to have Chrome either install For ease of use we recommend to run Chrome inside a Docker container: ```sh -docker pull alpeware/chrome-headless-trunk -docker run -d -p=0.0.0.0:9222:9222 --name=chrome-headless -v /tmp/chromedata/:/data alpeware/chrome-headless-trunk +docker pull alpeware/chrome-headless-stable +docker run -d -p=0.0.0.0:9222:9222 --name=chrome-headless -v /tmp/chromedata/:/data alpeware/chrome-headless-stable ``` But if you want to see what's happening during query execution, just start your Chrome with remote debugging port: @@ -207,14 +220,18 @@ import ( "context" "encoding/json" "fmt" - "github.com/MontFerret/ferret/pkg/compiler" "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" + "github.com/MontFerret/ferret/pkg/drivers/http" ) type Topic struct { Name string `json:"name"` Description string `json:"description"` - Url string `json:"url"` + URL string `json:"url"` } func main() { @@ -226,7 +243,7 @@ func main() { } for _, topic := range topics { - fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.Url)) + fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL)) } } @@ -255,7 +272,17 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return nil, err } - out, err := program.Run(context.Background()) + // create a root context + ctx := context.Background() + + // enable HTML drivers + // by default, Ferret Runtime does not know about any HTML drivers + // all HTML manipulations are done via functions from standard library + // that assume that at least one driver is available + ctx = drivers.WithContext(ctx, cdp.NewDriver()) + ctx = drivers.WithContext(ctx, http.NewDriver(), drivers.AsDefault()) + + out, err := program.Run(ctx) if err != nil { return nil, err @@ -271,6 +298,7 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return res, nil } + ``` ## Extensibility @@ -284,10 +312,12 @@ import ( "context" "encoding/json" "fmt" + "os" + "strings" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" - "os" ) func main() { @@ -307,7 +337,7 @@ func getStrings() ([]string, error) { // function implements is a type of a function that ferret supports as a runtime function transform := func(ctx context.Context, args ...core.Value) (core.Value, error) { // it's just a helper function which helps to validate a number of passed args - err := core.ValidateArgs(args, 1) + err := core.ValidateArgs(args, 1, 1) if err != nil { // it's recommended to return built-in None type, instead of nil @@ -324,7 +354,7 @@ func getStrings() ([]string, error) { // cast to built-in string type str := args[0].(values.String) - return str.Concat(values.NewString("_ferret")).ToUpper(), nil + return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil } query := ` @@ -334,7 +364,10 @@ func getStrings() ([]string, error) { ` comp := compiler.New() - comp.RegisterFunction("transform", transform) + + if err := comp.RegisterFunction("transform", transform); err != nil { + return nil, err + } program, err := comp.Compile(query) @@ -384,3 +417,127 @@ func main() { comp.RegisterFunctions(strings.NewLib()) } ``` + +## Proxy + +By default, Ferret does not use any proxies. Partially, due to inability to force Chrome/Chromium (or any other Chrome Devtools Protocol compatible browser) to use a prticular proxy. It should be done during a browser launch. + +But you can pass an address of a proxy server you want to use for static pages. + +#### CLI + +```sh +ferret --proxy=http://localhost:8888 my-query.fql +``` + +#### Code + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/http" +) + +func run(q string) ([]byte, error) { + proxy := "http://localhost:8888" + comp := compiler.New() + program := comp.MustCompile(q) + + // create a root context + ctx := context.Background() + + // we inform the driver what proxy to use + ctx = drivers.WithContext(ctx, http.NewDriver(http.WithProxy(proxy)), drivers.AsDefault()) + + return program.Run(ctx) +} + +``` + +## Cookies + +### Non-incognito mode + +By default, ``CDP`` driver execute each query in an incognito mode in order to avoid any collisions related to some persisted cookies from previous queries. +However, sometimes it might not be a desirable behavior and a query needs to be executed within a Chrome tab with earlier persisted cookies. +In order to do that, we need to inform the driver to execute all queries in regular tabs. Here is how to do that: + +#### CLI + +```sh +ferret --cdp-keep-cookies my-query.fql +``` + +#### Code + +```go +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" +) + +func run(q string) ([]byte, error) { + comp := compiler.New() + program := comp.MustCompile(q) + + // create a root context + ctx := context.Background() + + // we inform the driver to keep cookies between queries + ctx = drivers.WithContext( + ctx, + cdp.NewDriver(cdp.WithKeepCookies()), + drivers.AsDefault(), + ) + + return program.Run(ctx) +} +``` + +#### Query +``` +LET doc = DOCUMENT("https://www.google.com", { + driver: "cdp", + keepCookies: true +}) +``` + +### Cookies manipulation +For more precise work, you can set/get/delete cookies manually during and after page load: + +``` +LET doc = DOCUMENT("https://www.google.com", { + driver: "cdp", + cookies: [ + { + name: "foo", + value: "bar" + } + ] +}) + +COOKIES_SET(doc, { name: "baz", value: "qaz"}, { name: "daz", value: "gag" }) +COOKIES_DEL(doc, "foo") + +LET c = COOKIES_GET(doc, "baz") + +FOR cookie IN doc.cookies + RETURN cookie.name + +``` diff --git a/cli/autocompleter.go b/cli/autocompleter.go new file mode 100644 index 00000000..27c59f94 --- /dev/null +++ b/cli/autocompleter.go @@ -0,0 +1,50 @@ +package cli + +import ( + "strings" + + "github.com/derekparker/trie" +) + +// AutoCompleter autocompletes queries +// into the REPL. +// Implements AutoCompleter interface from +// github.com/chzyer/readline +type AutoCompleter struct { + coreFuncs *trie.Trie +} + +func NewAutoCompleter(functions []string) *AutoCompleter { + coreFuncs := trie.New() + + for _, function := range functions { + coreFuncs.Add(function, function) + } + + return &AutoCompleter{ + coreFuncs: coreFuncs, + } +} + +// Do implements method of AutoCompleter interface +func (ac *AutoCompleter) Do(line []rune, pos int) (newLine [][]rune, length int) { + lineStr := string(line) + tokens := strings.Split(lineStr, " ") + token := tokens[len(tokens)-1] + + // if remove this check, than + // on any empty string will return + // all available functions + if token == "" { + return newLine, pos + } + + for _, fn := range ac.coreFuncs.PrefixSearch(token) { + // cuts a piece of word that is already written + // in the repl + withoutPre := []rune(fn)[len(token):] + newLine = append(newLine, withoutPre) + } + + return newLine, pos +} diff --git a/cli/browser/browser.go b/cli/browser/browser.go index 84092e50..1593885c 100644 --- a/cli/browser/browser.go +++ b/cli/browser/browser.go @@ -1,10 +1,11 @@ package browser import ( - "github.com/pkg/errors" "os" "os/exec" "runtime" + + "github.com/pkg/errors" ) type Browser struct { @@ -18,7 +19,7 @@ func (b *Browser) Flags() Flags { func (b *Browser) DebuggingAddress() string { if !b.Flags().Has("remote-debugging-address") { - b.Flags().Set("remote-debugging-address", "0.0.0.0") + b.Flags().Set("remote-debugging-address", "http://0.0.0.0:9222") } value, _ := b.Flags().Get("remote-debugging-address") @@ -39,12 +40,16 @@ func (b *Browser) DebuggingPort() int { func (b *Browser) Close() error { var err error - if runtime.GOOS != "windows" { + if runtime.GOOS != goosWindows { err = b.cmd.Process.Signal(os.Interrupt) } else { err = b.cmd.Process.Kill() } + if err != nil { + return err + } + _, err = b.cmd.Process.Wait() if err != nil { diff --git a/cli/browser/flags.go b/cli/browser/flags.go index 47c6875a..7288e4d1 100644 --- a/cli/browser/flags.go +++ b/cli/browser/flags.go @@ -2,9 +2,10 @@ package browser import ( "fmt" - "github.com/pkg/errors" "sort" "strings" + + "github.com/pkg/errors" ) type Flags map[string]interface{} @@ -61,8 +62,6 @@ func (flags Flags) Has(arg string) bool { } func (flags Flags) List() []string { - var list []string - orderedFlags := make([]string, 0, 10) for arg := range flags { @@ -71,23 +70,25 @@ func (flags Flags) List() []string { sort.Strings(orderedFlags) - for _, arg := range orderedFlags { + list := make([]string, len(orderedFlags)) + + for i, arg := range orderedFlags { val, err := flags.Get(arg) if err != nil { continue } - switch val.(type) { + switch v := val.(type) { case int: - arg = fmt.Sprintf("--%s=%d", arg, val.(int)) + arg = fmt.Sprintf("--%s=%d", arg, v) case string: - arg = fmt.Sprintf("--%s=%s", arg, val.(string)) + arg = fmt.Sprintf("--%s=%s", arg, v) default: arg = fmt.Sprintf("--%s", arg) } - list = append(list, arg) + list[i] = arg } return list diff --git a/cli/browser/helpers.go b/cli/browser/helpers.go index 5631adad..818b82d1 100644 --- a/cli/browser/helpers.go +++ b/cli/browser/helpers.go @@ -1,29 +1,28 @@ package browser import ( - "fmt" "os" "os/exec" "runtime" ) -func resolveExecutablePath() string { - var res string +func resolveExecutablePath() (path string) { switch runtime.GOOS { - case "darwin": + case goosDarwin: for _, c := range []string{ "/Applications/Google Chrome Canary.app", "/Applications/Google Chrome.app", } { // MacOS apps are actually folders - if info, err := os.Stat(c); err == nil && info.IsDir() { - res = fmt.Sprintf("open %s -n", c) + info, err := os.Stat(c) + if err == nil && info.IsDir() { + path = c break } } - case "linux": + case goosLinux: for _, c := range []string{ "headless_shell", "chromium", @@ -31,13 +30,13 @@ func resolveExecutablePath() string { "google-chrome-unstable", "google-chrome-stable"} { if _, err := exec.LookPath(c); err == nil { - res = c + path = c break } } - case "windows": + case goosWindows: } - return res + return } diff --git a/cli/browser/launcher.go b/cli/browser/launcher.go index 8a41ba1c..afea06d2 100644 --- a/cli/browser/launcher.go +++ b/cli/browser/launcher.go @@ -1,12 +1,13 @@ package browser import ( - "github.com/pkg/errors" "io/ioutil" "os" "os/exec" "path/filepath" "runtime" + + "github.com/pkg/errors" ) func Launch(setters ...Option) (*Browser, error) { @@ -40,7 +41,7 @@ func Launch(setters ...Option) (*Browser, error) { flags.SetN("mute-audio") } - if runtime.GOOS == "windows" { + if runtime.GOOS == goosWindows { flags.SetN("disable-gpu") } @@ -78,7 +79,10 @@ func Launch(setters ...Option) (*Browser, error) { } } - cmd := exec.Command(chromeExecutable, flags.List()...) + execArgs := []string{chromeExecutable, "--args"} + execArgs = append(execArgs, flags.List()...) + + cmd := exec.Command("open", execArgs...) cmd.Dir = workDir err = cmd.Start() diff --git a/cli/browser/options.go b/cli/browser/options.go index 2f79a1ed..a714e6c9 100644 --- a/cli/browser/options.go +++ b/cli/browser/options.go @@ -18,6 +18,12 @@ type ( } ) +const ( + goosWindows = "windows" + goosLinux = "linux" + goosDarwin = "darwin" +) + func WithoutDefaultArgs() Option { return func(opts *Options) { opts.ignoreDefaultArgs = true diff --git a/cli/exec.go b/cli/exec.go index d3619383..14ffa970 100644 --- a/cli/exec.go +++ b/cli/exec.go @@ -9,7 +9,6 @@ import ( "io/ioutil" "os" "os/signal" - "syscall" ) func ExecFile(pathToFile string, opts Options) { @@ -38,9 +37,10 @@ func Exec(query string, opts Options) { l := NewLogger() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := opts.WithContext(context.Background()) + c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGHUP) + signal.Notify(c, os.Interrupt) go func() { for { @@ -59,12 +59,9 @@ func Exec(query string, opts Options) { out, err := prog.Run( ctx, - runtime.WithBrowser(opts.Cdp), runtime.WithLog(l), runtime.WithLogLevel(logging.DebugLevel), runtime.WithParams(opts.Params), - runtime.WithProxy(opts.Proxy), - runtime.WithUserAgent(opts.UserAgent), ) if opts.ShowTime { diff --git a/cli/options.go b/cli/options.go index 03667559..9804bd81 100644 --- a/cli/options.go +++ b/cli/options.go @@ -1,9 +1,50 @@ package cli +import ( + "context" + + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" + "github.com/MontFerret/ferret/pkg/drivers/http" +) + type Options struct { - Cdp string - Params map[string]interface{} - Proxy string - UserAgent string - ShowTime bool + Cdp string + Params map[string]interface{} + Proxy string + UserAgent string + ShowTime bool + KeepCookies bool +} + +func (opts Options) WithContext(ctx context.Context) (context.Context, context.CancelFunc) { + httpDriver := http.NewDriver( + http.WithProxy(opts.Proxy), + http.WithUserAgent(opts.UserAgent), + ) + + ctx = drivers.WithContext( + ctx, + httpDriver, + drivers.AsDefault(), + ) + + cdpOpts := []cdp.Option{ + cdp.WithAddress(opts.Cdp), + cdp.WithProxy(opts.Proxy), + cdp.WithUserAgent(opts.UserAgent), + } + + if opts.KeepCookies { + cdpOpts = append(cdpOpts, cdp.WithKeepCookies()) + } + + cdpDriver := cdp.NewDriver(cdpOpts...) + + ctx = drivers.WithContext( + ctx, + cdpDriver, + ) + + return context.WithCancel(ctx) } diff --git a/cli/repl.go b/cli/repl.go index e58b0efd..cd50472e 100644 --- a/cli/repl.go +++ b/cli/repl.go @@ -3,14 +3,16 @@ package cli import ( "context" "fmt" + "os" + "os/signal" + "strings" + + "github.com/MontFerret/ferret/pkg/parser/fql" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime" "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/chzyer/readline" - "os" - "os/signal" - "strings" - "syscall" ) func Repl(version string, opts Options) { @@ -23,6 +25,11 @@ func Repl(version string, opts Options) { Prompt: "> ", InterruptPrompt: "^C", EOFPrompt: "exit", + AutoComplete: NewAutoCompleter( + append( + fqlLiterals(), + ferret.RegisteredFunctions()..., + )), }) if err != nil { @@ -42,9 +49,10 @@ func Repl(version string, opts Options) { l := NewLogger() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := opts.WithContext(context.Background()) + c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGHUP) + signal.Notify(c, os.Interrupt) exit := func() { cancel() @@ -67,7 +75,7 @@ func Repl(version string, opts Options) { line = strings.TrimSpace(line) - if len(line) == 0 { + if line == "" { continue } @@ -110,12 +118,9 @@ func Repl(version string, opts Options) { out, err := program.Run( ctx, - runtime.WithBrowser(opts.Cdp), runtime.WithLog(l), runtime.WithLogLevel(logging.DebugLevel), runtime.WithParams(opts.Params), - runtime.WithProxy(opts.Proxy), - runtime.WithUserAgent(opts.UserAgent), ) if err != nil { @@ -132,3 +137,13 @@ func Repl(version string, opts Options) { } } } + +func fqlLiterals() (literals []string) { + lns := fql.NewFqlLexer(nil).LiteralNames + + for _, ln := range lns { + literals = append(literals, strings.Trim(ln, "'")) + } + + return +} diff --git a/e2e/main.go b/e2e/main.go index f47d2d5d..55c06caa 100644 --- a/e2e/main.go +++ b/e2e/main.go @@ -8,7 +8,9 @@ import ( "github.com/MontFerret/ferret/e2e/server" "github.com/rs/zerolog" "os" + "os/signal" "path/filepath" + "regexp" ) var ( @@ -24,17 +26,17 @@ var ( "root directory with test pages", ) - port = flag.Uint64( - "port", - 8080, - "server port", - ) - cdp = flag.String( "cdp", "http://0.0.0.0:9222", "address of remote Chrome instance", ) + + filter = flag.String( + "filter", + "", + "regexp expression to filter out tests", + ) ) func main() { @@ -42,17 +44,40 @@ func main() { logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}) - s := server.New(server.Settings{ - Port: *port, - Dir: *pagesDir, + staticPort := uint64(8080) + static := server.New(server.Settings{ + Port: staticPort, + Dir: filepath.Join(*pagesDir, "static"), }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + dynamicPort := uint64(8081) + dynamic := server.New(server.Settings{ + Port: dynamicPort, + Dir: filepath.Join(*pagesDir, "dynamic"), + }) + + var filterR *regexp.Regexp + + if *filter != "" { + r, err := regexp.Compile(*filter) + + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + filterR = r + } go func() { - if err := s.Start(); err != nil { - logger.Info().Timestamp().Msg("shutting down the server") + if err := static.Start(); err != nil { + logger.Info().Timestamp().Msg("shutting down the static pages server") + } + }() + + go func() { + if err := dynamic.Start(); err != nil { + logger.Info().Timestamp().Msg("shutting down the dynamic pages server") } }() @@ -71,18 +96,29 @@ func main() { } r := runner.New(logger, runner.Settings{ - ServerAddress: fmt.Sprintf("http://0.0.0.0:%d", *port), - CDPAddress: *cdp, - Dir: *testsDir, + StaticServerAddress: fmt.Sprintf("http://0.0.0.0:%d", staticPort), + DynamicServerAddress: fmt.Sprintf("http://0.0.0.0:%d", dynamicPort), + CDPAddress: *cdp, + Dir: *testsDir, + Filter: filterR, }) - err := r.Run() + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) - if err := s.Stop(ctx); err != nil { - logger.Fatal().Timestamp().Err(err).Msg("failed to stop server") - } + go func() { + for { + <-c + cancel() + } + }() + + err := r.Run(ctx) if err != nil { os.Exit(1) } + + os.Exit(0) } diff --git a/e2e/pages/bootstrap/overview.html b/e2e/pages/bootstrap/overview.html deleted file mode 100644 index 7b21a865..00000000 --- a/e2e/pages/bootstrap/overview.html +++ /dev/null @@ -1,565 +0,0 @@ - - - - - - - - - -Overview · Bootstrap - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Skip to main content -
- - - -
- Bootstrap - - -
- -
- - - - Download -
- - -
-
-
-
- - -
- - - -
- - -
- -
- - -
-

Overview

-

Components and options for laying out your Bootstrap project, including wrapping containers, a powerful grid system, a flexible media object, and responsive utility classes.

- - -

Containers

- -

Containers are the most basic layout element in Bootstrap and are required when using our default grid system. Choose from a responsive, fixed-width container (meaning its max-width changes at each breakpoint) or fluid-width (meaning it’s 100% wide all the time).

- -

While containers can be nested, most layouts do not require a nested container.

- -
-
-
-
-
-
-
- -
<div class="container">
-  <!-- Content here -->
-</div>
- -

Use .container-fluid for a full width container, spanning the entire width of the viewport.

- -
-
-
-
-
-
-
- -
<div class="container-fluid">
-  ...
-</div>
- -

Responsive breakpoints

- -

Since Bootstrap is developed to be mobile first, we use a handful of media queries to create sensible breakpoints for our layouts and interfaces. These breakpoints are mostly based on minimum viewport widths and allow us to scale up elements as the viewport changes.

- -

Bootstrap primarily uses the following media query ranges—or breakpoints—in our source Sass files for our layout, grid system, and components.

- -
// Extra small devices (portrait phones, less than 576px)
-// No media query for `xs` since this is the default in Bootstrap
-
-// Small devices (landscape phones, 576px and up)
-@media (min-width: 576px) { ... }
-
-// Medium devices (tablets, 768px and up)
-@media (min-width: 768px) { ... }
-
-// Large devices (desktops, 992px and up)
-@media (min-width: 992px) { ... }
-
-// Extra large devices (large desktops, 1200px and up)
-@media (min-width: 1200px) { ... }
- -

Since we write our source CSS in Sass, all our media queries are available via Sass mixins:

- -
// No media query necessary for xs breakpoint as it's effectively `@media (min-width: 0) { ... }`
-@include media-breakpoint-up(sm) { ... }
-@include media-breakpoint-up(md) { ... }
-@include media-breakpoint-up(lg) { ... }
-@include media-breakpoint-up(xl) { ... }
-
-// Example: Hide starting at `min-width: 0`, and then show at the `sm` breakpoint
-.custom-class {
-  display: none;
-}
-@include media-breakpoint-up(sm) {
-  .custom-class {
-    display: block;
-  }
-}
- -

We occasionally use media queries that go in the other direction (the given screen size or smaller):

- -
// Extra small devices (portrait phones, less than 576px)
-@media (max-width: 575.98px) { ... }
-
-// Small devices (landscape phones, less than 768px)
-@media (max-width: 767.98px) { ... }
-
-// Medium devices (tablets, less than 992px)
-@media (max-width: 991.98px) { ... }
-
-// Large devices (desktops, less than 1200px)
-@media (max-width: 1199.98px) { ... }
-
-// Extra large devices (large desktops)
-// No media query since the extra-large breakpoint has no upper bound on its width
- -
-

Note that since browsers do not currently support range context queries, we work around the limitations of min- and max- prefixes and viewports with fractional widths (which can occur under certain conditions on high-dpi devices, for instance) by using values with higher precision for these comparisons.

-
- -

Once again, these media queries are also available via Sass mixins:

- -
@include media-breakpoint-down(xs) { ... }
-@include media-breakpoint-down(sm) { ... }
-@include media-breakpoint-down(md) { ... }
-@include media-breakpoint-down(lg) { ... }
-// No media query necessary for xl breakpoint as it has no upper bound on its width
-
-// Example: Style from medium breakpoint and down
-@include media-breakpoint-down(md) {
-  .custom-class {
-    display: block;
-  }
-}
- -

There are also media queries and mixins for targeting a single segment of screen sizes using the minimum and maximum breakpoint widths.

- -
// Extra small devices (portrait phones, less than 576px)
-@media (max-width: 575.98px) { ... }
-
-// Small devices (landscape phones, 576px and up)
-@media (min-width: 576px) and (max-width: 767.98px) { ... }
-
-// Medium devices (tablets, 768px and up)
-@media (min-width: 768px) and (max-width: 991.98px) { ... }
-
-// Large devices (desktops, 992px and up)
-@media (min-width: 992px) and (max-width: 1199.98px) { ... }
-
-// Extra large devices (large desktops, 1200px and up)
-@media (min-width: 1200px) { ... }
- -

These media queries are also available via Sass mixins:

- -
@include media-breakpoint-only(xs) { ... }
-@include media-breakpoint-only(sm) { ... }
-@include media-breakpoint-only(md) { ... }
-@include media-breakpoint-only(lg) { ... }
-@include media-breakpoint-only(xl) { ... }
- -

Similarly, media queries may span multiple breakpoint widths:

- -
// Example
-// Apply styles starting from medium devices and up to extra large devices
-@media (min-width: 768px) and (max-width: 1199.98px) { ... }
- -

The Sass mixin for targeting the same screen size range would be:

- -
@include media-breakpoint-between(md, xl) { ... }
- -

Z-index

- -

Several Bootstrap components utilize z-index, the CSS property that helps control layout by providing a third axis to arrange content. We utilize a default z-index scale in Bootstrap that’s been designed to properly layer navigation, tooltips and popovers, modals, and more.

- -

These higher values start at an arbitrary number, high and specific enough to ideally avoid conflicts. We need a standard set of these across our layered components—tooltips, popovers, navbars, dropdowns, modals—so we can be reasonably consistent in the behaviors. There’s no reason we couldn’t have used 100+ or 500+.

- -

We don’t encourage customization of these individual values; should you change one, you likely need to change them all.

- -
$zindex-dropdown:          1000 !default;
-$zindex-sticky:            1020 !default;
-$zindex-fixed:             1030 !default;
-$zindex-modal-backdrop:    1040 !default;
-$zindex-modal:             1050 !default;
-$zindex-popover:           1060 !default;
-$zindex-tooltip:           1070 !default;
- -

To handle overlapping borders within components (e.g., buttons and inputs in input groups), we use low single digit z-index values of 1, 2, and 3 for default, hover, and active states. On hover/focus/active, we bring a particular element to the forefront with a higher z-index value to show their border over the sibling elements.

- -
-
-
- - - - - - - - \ No newline at end of file diff --git a/e2e/pages/dynamic/components/app.js b/e2e/pages/dynamic/components/app.js new file mode 100644 index 00000000..c7338a62 --- /dev/null +++ b/e2e/pages/dynamic/components/app.js @@ -0,0 +1,34 @@ +import Layout from './layout.js'; +import IndexPage from './pages/index.js'; +import FormsPage from './pages/forms/index.js'; +import EventsPage from './pages/events/index.js'; + +const e = React.createElement; +const Router = ReactRouter.Router; +const Switch = ReactRouter.Switch; +const Route = ReactRouter.Route; +const Redirect = ReactRouter.Redirect; +const createBrowserHistory = History.createBrowserHistory; + +export default function AppComponent({ redirect = null}) { + return e(Router, { history: createBrowserHistory() }, + e(Layout, null, [ + e(Switch, null, [ + e(Route, { + path: '/', + exact: true, + component: IndexPage + }), + e(Route, { + path: '/forms', + component: FormsPage + }), + e(Route, { + path: '/events', + component: EventsPage + }), + ]), + redirect ? e(Redirect, { to: redirect }) : null + ]) + ) +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/layout.js b/e2e/pages/dynamic/components/layout.js new file mode 100644 index 00000000..3ae6fbf1 --- /dev/null +++ b/e2e/pages/dynamic/components/layout.js @@ -0,0 +1,27 @@ +const e = React.createElement; +const NavLink = ReactRouterDOM.NavLink; + +export default function Layout({ children }) { + return e("div", { id: "layout"}, [ + e("nav", { className: "navbar navbar-expand-md navbar-dark bg-dark mb-4", id: "navbar" }, [ + e(NavLink, { className: "navbar-brand", to: "/"}, "Ferret"), + e("button", { className: "navbar-toggler", type: "button"}, [ + e("span", { className: "navbar-toggler-icon" }) + ]), + e("div", { className: "collapse navbar-collapse" }, [ + e("ul", { className: "navbar-nav mr-auto" }, [ + e("li", { className: "nav-item"}, [ + e(NavLink, { className: "nav-link", to: "/forms" }, "Forms") + ]), + e("li", { className: "nav-item"}, [ + e(NavLink, { className: "nav-link", to: "/navigation" }, "Navigation") + ]), + e("li", { className: "nav-item"}, [ + e(NavLink, { className: "nav-link", to: "/events" }, "Events") + ]) + ]) + ]) + ]), + e("main", { className: "container"}, children) + ]) +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/events/appearable.js b/e2e/pages/dynamic/components/pages/events/appearable.js new file mode 100644 index 00000000..160fd0ed --- /dev/null +++ b/e2e/pages/dynamic/components/pages/events/appearable.js @@ -0,0 +1,71 @@ +import random from "../../../utils/random.js"; + +const e = React.createElement; + +function render(id, props = {}) { + return e("span", { id: `${id}-content`, ...props }, ["Hello world"]); +} + +export default class AppearableComponent extends React.PureComponent { + constructor(props) { + super(props); + + let element = null; + + if (props.appear) { + if (props.useStyle) { + element = render(props.id, { style: {display: "none"}}) + } + } else { + if (props.useStyle) { + element = render(props.id, { style: {display: "block" }}) + } else { + element = render(props.id) + } + } + + this.state = { + element + }; + } + + handleClick() { + setTimeout(() => { + const props = this.props; + let element = null; + + if (props.appear) { + if (props.useStyle) { + element = render(props.id, { style: {display: "block" }}) + } else { + element = render(props.id) + } + } else { + if (props.useStyle) { + element = render(props.id, { style: {display: "none"}}) + } + } + + this.setState({ + element, + }) + }, random()) + } + + render() { + const btnId = `${this.props.id}-btn`; + + return e("div", {className: "card"}, [ + e("div", { className: "card-header"}, [ + e("button", { + id: btnId, + className: "btn btn-primary", + onClick: this.handleClick.bind(this) + }, [ + this.props.title || "Toggle class" + ]) + ]), + e("div", { className: "card-body"}, this.state.element) + ]); + } +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/events/clickable.js b/e2e/pages/dynamic/components/pages/events/clickable.js new file mode 100644 index 00000000..f0d3b48c --- /dev/null +++ b/e2e/pages/dynamic/components/pages/events/clickable.js @@ -0,0 +1,56 @@ +import random from "../../../utils/random.js"; + +const e = React.createElement; + +export default class ClickableComponent extends React.PureComponent { + constructor(props) { + super(props); + + this.state = { + show: props.show === true + }; + } + + handleClick() { + let timeout = 500; + + if (this.props.randomTimeout) { + timeout = random(); + } + + setTimeout(() => { + this.setState({ + show: !this.state.show + }) + }, timeout) + } + + render() { + const btnId = `${this.props.id}-btn`; + const contentId = `${this.props.id}-content`; + const classNames = ["alert"]; + + if (this.state.show === true) { + classNames.push("alert-success"); + } + + return e("div", {className: "card clickable"}, [ + e("div", { className: "card-header"}, [ + e("button", { + id: btnId, + className: "btn btn-primary", + onClick: this.handleClick.bind(this) + }, [ + this.props.title || "Toggle class" + ]) + ]), + e("div", { className: "card-body"}, [ + e("div", { id: contentId, className: classNames.join(" ")}, [ + e("p", null, [ + "Lorem ipsum dolor sit amet." + ]) + ]) + ]) + ]); + } +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/events/hoverable.js b/e2e/pages/dynamic/components/pages/events/hoverable.js new file mode 100644 index 00000000..ded13f5e --- /dev/null +++ b/e2e/pages/dynamic/components/pages/events/hoverable.js @@ -0,0 +1,47 @@ +const e = React.createElement; + +export default class HoverableComponent extends React.PureComponent { + constructor(props) { + super(props); + + this.state = { + hovered: false + }; + } + + handleMouseEnter() { + this.setState({ + hovered: true + }); + } + + handleMouseLeave() { + this.setState({ + hovered: false + }); + } + + render() { + let content; + + if (this.state.hovered) { + content = e("p", { id: "hoverable-content"}, [ + "Lorem ipsum dolor sit amet." + ]); + } + + return e("div", { className: "card"}, [ + e("div", {className: "card-header"}, [ + e("button", { + id: "hoverable-btn", + className: "btn btn-primary", + onMouseEnter: this.handleMouseEnter.bind(this), + onMouseLeave: this.handleMouseLeave.bind(this) + }, [ + "Show content" + ]) + ]), + e("div", {className: "card-body"}, content) + ]); + } +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/events/index.js b/e2e/pages/dynamic/components/pages/events/index.js new file mode 100644 index 00000000..c05dbe8a --- /dev/null +++ b/e2e/pages/dynamic/components/pages/events/index.js @@ -0,0 +1,79 @@ +import Hoverable from "./hoverable.js"; +import Clickable from "./clickable.js"; +import Appearable from "./appearable.js"; + +const e = React.createElement; + +export default class EventsPage extends React.Component { + render() { + return e("div", { id: "page-events" }, [ + e("div", { className: "row" }, [ + e("div", { className: "col-lg-4"}, [ + e(Hoverable), + ]), + e("div", { className: "col-lg-4"}, [ + e(Clickable, { + id: "wait-class", + title: "Add class" + }) + ]), + e("div", { className: "col-lg-4"}, [ + e(Clickable, { + id: "wait-class-random", + title: "Add class 2", + randomTimeout: true + }) + ]) + ]), + e("div", { className: "row" }, [ + e("div", { className: "col-lg-4"}, [ + e(Clickable, { + id: "wait-no-class", + title: "Remove class", + show: true + }) + ]), + e("div", { className: "col-lg-4"}, [ + e(Clickable, { + id: "wait-no-class-random", + title: "Remove class 2", + show: true, + randomTimeout: true + }) + ]), + e("div", { className: "col-lg-4"}, [ + e(Appearable, { + id: "wait-element", + appear: true, + title: "Appearable" + }) + ]), + ]), + e("div", { className: "row" }, [ + e("div", { className: "col-lg-4"}, [ + e(Appearable, { + id: "wait-no-element", + appear: false, + title: "Disappearable" + }) + ]), + e("div", { className: "col-lg-4"}, [ + e(Appearable, { + id: "wait-style", + appear: true, + title: "Appearable with style", + useStyle: true, + }) + ]), + e("div", { className: "col-lg-4"}, [ + e(Appearable, { + id: "wait-no-style", + appear: false, + title: "Disappearable", + useStyle: true, + }) + ]), + ]) + ]) + } +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/forms/index.js b/e2e/pages/dynamic/components/pages/forms/index.js new file mode 100644 index 00000000..5c8ae785 --- /dev/null +++ b/e2e/pages/dynamic/components/pages/forms/index.js @@ -0,0 +1,124 @@ +const e = React.createElement; + +export default class FormsPage extends React.Component { + constructor(props) { + super(props); + + this.state = { + textInput: "", + select: "", + multiSelect: "", + textarea: "" + }; + + this.handleTextInput = (evt) => { + evt.preventDefault(); + + this.setState({ + textInput: evt.target.value + }); + }; + + this.handleSelect = (evt) => { + evt.preventDefault(); + + this.setState({ + select: evt.target.value + }); + }; + + this.handleMultiSelect = (evt) => { + evt.preventDefault(); + + this.setState({ + multiSelect: Array.prototype.map.call(evt.target.selectedOptions, i => i.value).join(", ") + }); + }; + + this.handleTtextarea = (evt) => { + evt.preventDefault(); + + this.setState({ + textarea: evt.target.value + }); + } + } + + render() { + return e("form", { id: "page-form" }, [ + e("div", { className: "form-group" }, [ + e("label", null, "Text input"), + e("input", { + id: "text_input", + type: "text", + className: "form-control", + onChange: this.handleTextInput + }), + e("small", { + id: "text_output", + className: "form-text text-muted" + }, + this.state.textInput + ) + ]), + e("div", { className: "form-group" }, [ + e("label", null, "Select"), + e("select", { + id: "select_input", + className: "form-control", + onChange: this.handleSelect + }, + [ + e("option", null, 1), + e("option", null, 2), + e("option", null, 3), + e("option", null, 4), + e("option", null, 5), + ] + ), + e("small", { + id: "select_output", + className: "form-text text-muted" + }, this.state.select + ) + ]), + e("div", { className: "form-group" }, [ + e("label", null, "Multi select"), + e("select", { + id: "multi_select_input", + multiple: true, + className: "form-control", + onChange: this.handleMultiSelect + }, + [ + e("option", null, 1), + e("option", null, 2), + e("option", null, 3), + e("option", null, 4), + e("option", null, 5), + ] + ), + e("small", { + id: "multi_select_output", + className: "form-text text-muted" + }, this.state.multiSelect + ) + ]), + e("div", { className: "form-group" }, [ + e("label", null, "Textarea"), + e("textarea", { + id: "textarea_input", + rows:"5", + className: "form-control", + onChange: this.handleTtextarea + } + ), + e("small", { + id: "textarea_output", + className: "form-text text-muted" + }, this.state.textarea + ) + ]), + ]) + } +} \ No newline at end of file diff --git a/e2e/pages/dynamic/components/pages/index.js b/e2e/pages/dynamic/components/pages/index.js new file mode 100644 index 00000000..eb2372d8 --- /dev/null +++ b/e2e/pages/dynamic/components/pages/index.js @@ -0,0 +1,12 @@ +const e = React.createElement; + +export default function IndexPage() { + return e("div", { className: "jumbotron", "data-type": "page", id: "index" }, [ + e("div", null, + e("h1", null, "Welcome to Ferret E2E test page!") + ), + e("div", null, + e("p", { className: "lead" }, "It has several pages for testing different possibilities of the library") + ) + ]) +} \ No newline at end of file diff --git a/e2e/pages/dynamic/index.css b/e2e/pages/dynamic/index.css new file mode 100644 index 00000000..c018d8a7 --- /dev/null +++ b/e2e/pages/dynamic/index.css @@ -0,0 +1,4 @@ +/* Show it's not fixed to the top */ +body { + min-height: 75rem; +} diff --git a/e2e/pages/dynamic/index.html b/e2e/pages/dynamic/index.html new file mode 100644 index 00000000..d854a8b9 --- /dev/null +++ b/e2e/pages/dynamic/index.html @@ -0,0 +1,21 @@ + + + + + + Ferret E2E SPA + + + + + + +
+ + + + + + + + diff --git a/e2e/pages/dynamic/index.js b/e2e/pages/dynamic/index.js new file mode 100644 index 00000000..7edcf959 --- /dev/null +++ b/e2e/pages/dynamic/index.js @@ -0,0 +1,9 @@ +import AppComponent from "./components/app.js"; +import { parse } from "./utils/qs.js"; + +const qs = parse(location.search); + +ReactDOM.render( + React.createElement(AppComponent, qs), + document.getElementById("root") +); diff --git a/e2e/pages/dynamic/utils/qs.js b/e2e/pages/dynamic/utils/qs.js new file mode 100644 index 00000000..76042158 --- /dev/null +++ b/e2e/pages/dynamic/utils/qs.js @@ -0,0 +1,82 @@ +'use strict'; + +var has = Object.prototype.hasOwnProperty + , undef; + +/** + * Decode a URI encoded string. + * + * @param {String} input The URI encoded string. + * @returns {String} The decoded string. + * @api private + */ +function decode(input) { + return decodeURIComponent(input.replace(/\+/g, ' ')); +} + +/** + * Simple query string parser. + * + * @param {String} query The query string that needs to be parsed. + * @returns {Object} + * @api public + */ +export function parse(query) { + var parser = /([^=?&]+)=?([^&]*)/g + , result = {} + , part; + + while (part = parser.exec(query)) { + var key = decode(part[1]) + , value = decode(part[2]); + + // + // Prevent overriding of existing properties. This ensures that build-in + // methods like `toString` or __proto__ are not overriden by malicious + // querystrings. + // + if (key in result) continue; + result[key] = value; + } + + return result; +} + +/** + * Transform a query string to an object. + * + * @param {Object} obj Object that should be transformed. + * @param {String} prefix Optional prefix. + * @returns {String} + * @api public + */ +export function stringify(obj, prefix) { + prefix = prefix || ''; + + var pairs = [] + , value + , key; + + // + // Optionally prefix with a '?' if needed + // + if ('string' !== typeof prefix) prefix = '?'; + + for (key in obj) { + if (has.call(obj, key)) { + value = obj[key]; + + // + // Edge cases where we actually want to encode the value to an empty + // string instead of the stringified value. + // + if (!value && (value === null || value === undef || isNaN(value))) { + value = ''; + } + + pairs.push(encodeURIComponent(key) +'='+ encodeURIComponent(value)); + } + } + + return pairs.length ? prefix + pairs.join('&') : ''; +} \ No newline at end of file diff --git a/e2e/pages/dynamic/utils/random.js b/e2e/pages/dynamic/utils/random.js new file mode 100644 index 00000000..2f401403 --- /dev/null +++ b/e2e/pages/dynamic/utils/random.js @@ -0,0 +1,13 @@ +export default function random(min = 1000, max = 5000) { + const val = Math.random() * 1000 * 10; + + if (val < min) { + return min; + } + + if (val > max) { + return max; + } + + return val; +} \ No newline at end of file diff --git a/e2e/pages/bootstrap/assets/analytics.js b/e2e/pages/static/assets/analytics.js similarity index 100% rename from e2e/pages/bootstrap/assets/analytics.js rename to e2e/pages/static/assets/analytics.js diff --git a/e2e/pages/bootstrap/assets/bootstrap.min.css b/e2e/pages/static/assets/bootstrap.min.css similarity index 100% rename from e2e/pages/bootstrap/assets/bootstrap.min.css rename to e2e/pages/static/assets/bootstrap.min.css diff --git a/e2e/pages/bootstrap/assets/bootstrap.min.js b/e2e/pages/static/assets/bootstrap.min.js similarity index 100% rename from e2e/pages/bootstrap/assets/bootstrap.min.js rename to e2e/pages/static/assets/bootstrap.min.js diff --git a/e2e/pages/bootstrap/assets/carbon.js b/e2e/pages/static/assets/carbon.js similarity index 100% rename from e2e/pages/bootstrap/assets/carbon.js rename to e2e/pages/static/assets/carbon.js diff --git a/e2e/pages/bootstrap/assets/docs.min.css b/e2e/pages/static/assets/docs.min.css similarity index 100% rename from e2e/pages/bootstrap/assets/docs.min.css rename to e2e/pages/static/assets/docs.min.css diff --git a/e2e/pages/bootstrap/assets/docs.min.js b/e2e/pages/static/assets/docs.min.js similarity index 100% rename from e2e/pages/bootstrap/assets/docs.min.js rename to e2e/pages/static/assets/docs.min.js diff --git a/e2e/pages/bootstrap/assets/docsearch.min.css b/e2e/pages/static/assets/docsearch.min.css similarity index 100% rename from e2e/pages/bootstrap/assets/docsearch.min.css rename to e2e/pages/static/assets/docsearch.min.css diff --git a/e2e/pages/bootstrap/assets/docsearch.min.js b/e2e/pages/static/assets/docsearch.min.js similarity index 100% rename from e2e/pages/bootstrap/assets/docsearch.min.js rename to e2e/pages/static/assets/docsearch.min.js diff --git a/e2e/pages/bootstrap/assets/jquery-3.3.1.slim.min.js b/e2e/pages/static/assets/jquery-3.3.1.slim.min.js similarity index 100% rename from e2e/pages/bootstrap/assets/jquery-3.3.1.slim.min.js rename to e2e/pages/static/assets/jquery-3.3.1.slim.min.js diff --git a/e2e/pages/bootstrap/assets/popper.min.js b/e2e/pages/static/assets/popper.min.js similarity index 100% rename from e2e/pages/bootstrap/assets/popper.min.js rename to e2e/pages/static/assets/popper.min.js diff --git a/e2e/pages/bootstrap/grid.html b/e2e/pages/static/grid.html similarity index 100% rename from e2e/pages/bootstrap/grid.html rename to e2e/pages/static/grid.html diff --git a/e2e/pages/bootstrap/media.html b/e2e/pages/static/media.html similarity index 100% rename from e2e/pages/bootstrap/media.html rename to e2e/pages/static/media.html diff --git a/e2e/pages/static/overview.html b/e2e/pages/static/overview.html new file mode 100644 index 00000000..e44f7682 --- /dev/null +++ b/e2e/pages/static/overview.html @@ -0,0 +1,1128 @@ + + + + + + + + + + Overview · Bootstrap + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to main content +
+ +
+ + + Bootstrap + + + + +
+ +
+ + Download +
+
+
+
+
+ + + + + + +
+ +
+
+ +
+
+

Overview

+

Components and options for laying out your Bootstrap project, including wrapping containers, a powerful grid system, a flexible media object, and responsive utility classes.

+ + +

+
Containers + +
+

+

Containers are the most basic layout element in Bootstrap and are + required when using our default grid system. Choose from a responsive, fixed-width container (meaning its + max-width changes at each breakpoint) or fluid-width (meaning it’s + 100% wide all the time). +

+

While containers + can be nested, most layouts do not require a nested container. +

+
+
+
+
+
+
+
+
+ +
+
+ 
+																																									
+																																										<div
+																																										class=
+																																										"container"
+																																										>
+																																										<!-- Content here -->
+																																										</div>
+																																									
+
+
+

Use + .container-fluid for a full width container, spanning the entire width of the viewport. +

+
+
+
+
+
+
+
+
+ +
+
+ 
+																																									
+																																										<div
+																																										class=
+																																										"container-fluid"
+																																										>
+  ...
+
+																																										</div>
+																																									
+
+
+

+
Responsive breakpoints + +
+

+

Since Bootstrap is developed to be mobile first, we use a handful of + media queries to create sensible breakpoints for our layouts and interfaces. These breakpoints are mostly based on minimum viewport widths and allow us to scale up elements as the viewport changes. +

+

Bootstrap primarily uses the following media query ranges—or breakpoints—in our source Sass files for our layout, grid system, and components.

+
+ +
+
+ 
+																																									
+																																										// Extra small devices (portrait phones, less than 576px)
+																																										// No media query for `xs` since this is the default in Bootstrap
+																																										// Small devices (landscape phones, 576px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										576px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Medium devices (tablets, 768px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										768px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Large devices (desktops, 992px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										992px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Extra large devices (large desktops, 1200px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										1200px
+																																										)
+																																										{
+																																										...
+																																										}
+																																									
+
+
+

Since we write our source CSS in Sass, all our media queries are available via Sass mixins:

+
+ +
+
+ 
+																																									
+																																										// No media query necessary for xs breakpoint as it's effectively `@media (min-width: 0) { ... }`
+																																										@include
+																																										media-breakpoint-up
+																																										(
+																																										sm
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-up
+																																										(
+																																										md
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-up
+																																										(
+																																										lg
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-up
+																																										(
+																																										xl
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Example: Hide starting at `min-width: 0`, and then show at the `sm` breakpoint
+																																										.custom-class
+																																										{
+																																										display
+																																										:
+																																										none
+																																										;
+																																										}
+																																										@include
+																																										media-breakpoint-up
+																																										(
+																																										sm
+																																										)
+																																										{
+																																										.custom-class
+																																										{
+																																										display
+																																										:
+																																										block
+																																										;
+																																										}
+																																										}
+																																									
+
+
+

We occasionally use media queries that go in the other direction (the given screen size + or smaller): +

+
+ +
+
+ 
+																																									
+																																										// Extra small devices (portrait phones, less than 576px)
+																																										@media
+																																										(
+																																										max-width
+																																										:
+																																										575
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Small devices (landscape phones, less than 768px)
+																																										@media
+																																										(
+																																										max-width
+																																										:
+																																										767
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Medium devices (tablets, less than 992px)
+																																										@media
+																																										(
+																																										max-width
+																																										:
+																																										991
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Large devices (desktops, less than 1200px)
+																																										@media
+																																										(
+																																										max-width
+																																										:
+																																										1199
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Extra large devices (large desktops)
+																																										// No media query since the extra-large breakpoint has no upper bound on its width
+																																									
+
+
+
+

Note that since browsers do not currently support + range context queries, we work around the limitations of + + min- and + max- prefixes + and viewports with fractional widths (which can occur under certain conditions on high-dpi devices, for instance) by using values with higher precision for these comparisons. +

+
+

Once again, these media queries are also available via Sass mixins:

+
+ +
+
+ 
+																																									
+																																										@include
+																																										media-breakpoint-down
+																																										(
+																																										xs
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-down
+																																										(
+																																										sm
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-down
+																																										(
+																																										md
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-down
+																																										(
+																																										lg
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// No media query necessary for xl breakpoint as it has no upper bound on its width
+																																										// Example: Style from medium breakpoint and down
+																																										@include
+																																										media-breakpoint-down
+																																										(
+																																										md
+																																										)
+																																										{
+																																										.custom-class
+																																										{
+																																										display
+																																										:
+																																										block
+																																										;
+																																										}
+																																										}
+																																									
+
+
+

There are also media queries and mixins for targeting a single segment of screen sizes using the minimum and maximum breakpoint widths.

+
+ +
+
+ 
+																																									
+																																										// Extra small devices (portrait phones, less than 576px)
+																																										@media
+																																										(
+																																										max-width
+																																										:
+																																										575
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Small devices (landscape phones, 576px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										576px
+																																										)
+																																										and
+																																										(
+																																										max-width
+																																										:
+																																										767
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Medium devices (tablets, 768px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										768px
+																																										)
+																																										and
+																																										(
+																																										max-width
+																																										:
+																																										991
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Large devices (desktops, 992px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										992px
+																																										)
+																																										and
+																																										(
+																																										max-width
+																																										:
+																																										1199
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																										// Extra large devices (large desktops, 1200px and up)
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										1200px
+																																										)
+																																										{
+																																										...
+																																										}
+																																									
+
+
+

These media queries are also available via Sass mixins:

+
+ +
+
+ 
+																																									
+																																										@include
+																																										media-breakpoint-only
+																																										(
+																																										xs
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-only
+																																										(
+																																										sm
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-only
+																																										(
+																																										md
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-only
+																																										(
+																																										lg
+																																										)
+																																										{
+																																										...
+																																										}
+																																										@include
+																																										media-breakpoint-only
+																																										(
+																																										xl
+																																										)
+																																										{
+																																										...
+																																										}
+																																									
+
+
+

Similarly, media queries may span multiple breakpoint widths:

+
+ +
+
+ 
+																																									
+																																										// Example
+																																										// Apply styles starting from medium devices and up to extra large devices
+																																										@media
+																																										(
+																																										min-width
+																																										:
+																																										768px
+																																										)
+																																										and
+																																										(
+																																										max-width
+																																										:
+																																										1199
+																																										.98px
+																																										)
+																																										{
+																																										...
+																																										}
+																																									
+
+
+

The Sass mixin for targeting the same screen size range would be:

+
+ +
+
+ 
+																																									
+																																										@include
+																																										media-breakpoint-between
+																																										(
+																																										md
+																																										,
+																																										xl
+																																										)
+																																										{
+																																										...
+																																										}
+																																									
+
+
+

+
Z-index + +
+

+

Several Bootstrap components utilize + z-index, the CSS property that helps control layout by providing a third axis to arrange content. We utilize a default z-index scale in Bootstrap that’s been designed to properly layer navigation, tooltips and popovers, modals, and more. +

+

These higher values start at an arbitrary number, high and specific enough to ideally avoid conflicts. We need a standard set of these across our layered components—tooltips, popovers, navbars, dropdowns, modals—so we can be reasonably consistent in the behaviors. There’s no reason we couldn’t have used + 100+ or + 500+. +

+

We don’t encourage customization of these individual values; should you change one, you likely need to change them all.

+
+ +
+
+ 
+																																									
+																																										$zindex-dropdown
+																																										:
+																																										1000
+																																										!
+																																										default
+																																										;
+																																										$zindex-sticky
+																																										:
+																																										1020
+																																										!
+																																										default
+																																										;
+																																										$zindex-fixed
+																																										:
+																																										1030
+																																										!
+																																										default
+																																										;
+																																										$zindex-modal-backdrop
+																																										:
+																																										1040
+																																										!
+																																										default
+																																										;
+																																										$zindex-modal
+																																										:
+																																										1050
+																																										!
+																																										default
+																																										;
+																																										$zindex-popover
+																																										:
+																																										1060
+																																										!
+																																										default
+																																										;
+																																										$zindex-tooltip
+																																										:
+																																										1070
+																																										!
+																																										default
+																																										;
+																																									
+
+
+

To handle overlapping borders within components (e.g., buttons and inputs in input groups), we use low single digit + z-index values of + 1, + 2, and + 3 for default, hover, and active states. On hover/focus/active, we bring a particular element to the forefront with a higher + z-index value to show their border over the sibling elements. +

+
+
+
+ + + + + + + + \ No newline at end of file diff --git a/e2e/pages/static/simple.html b/e2e/pages/static/simple.html new file mode 100644 index 00000000..0c9c0ba4 --- /dev/null +++ b/e2e/pages/static/simple.html @@ -0,0 +1,10 @@ + + + + + Title + + +

Hello world

+ + \ No newline at end of file diff --git a/e2e/pages/bootstrap/utilities.html b/e2e/pages/static/utilities.html similarity index 100% rename from e2e/pages/bootstrap/utilities.html rename to e2e/pages/static/utilities.html diff --git a/e2e/pages/static/value.html b/e2e/pages/static/value.html new file mode 100644 index 00000000..63eec22e --- /dev/null +++ b/e2e/pages/static/value.html @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
some column
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
foobar
+ + \ No newline at end of file diff --git a/e2e/runner/lib.go b/e2e/runner/lib.go index ea7d3858..e3e3ef01 100644 --- a/e2e/runner/lib.go +++ b/e2e/runner/lib.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" ) @@ -24,5 +25,5 @@ func expect(_ context.Context, args ...core.Value) (core.Value, error) { return values.EmptyString, nil } - return values.NewString(fmt.Sprintf(`expected "%s"", but got "%s"`, args[0], args[1])), nil + return values.NewString(fmt.Sprintf(`expected "%s", but got "%s"`, args[0], args[1])), nil } diff --git a/e2e/runner/runner.go b/e2e/runner/runner.go index e82c2486..389781eb 100644 --- a/e2e/runner/runner.go +++ b/e2e/runner/runner.go @@ -3,20 +3,28 @@ package runner import ( "context" "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "time" + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" + "github.com/MontFerret/ferret/pkg/drivers/http" "github.com/MontFerret/ferret/pkg/runtime" "github.com/pkg/errors" "github.com/rs/zerolog" - "io/ioutil" - "path/filepath" - "time" ) type ( Settings struct { - ServerAddress string - CDPAddress string - Dir string + StaticServerAddress string + DynamicServerAddress string + CDPAddress string + Dir string + Filter *regexp.Regexp } Result struct { @@ -44,8 +52,19 @@ func New(logger zerolog.Logger, settings Settings) *Runner { } } -func (r *Runner) Run() error { - results, err := r.runQueries(r.settings.Dir) +func (r *Runner) Run(ctx context.Context) error { + ctx = drivers.WithContext( + ctx, + cdp.NewDriver(cdp.WithAddress(r.settings.CDPAddress)), + ) + + ctx = drivers.WithContext( + ctx, + http.NewDriver(), + drivers.AsDefault(), + ) + + results, err := r.runQueries(ctx, r.settings.Dir) if err != nil { return err @@ -75,7 +94,7 @@ func (r *Runner) Run() error { return nil } -func (r *Runner) runQueries(dir string) ([]Result, error) { +func (r *Runner) runQueries(ctx context.Context, dir string) ([]Result, error) { files, err := ioutil.ReadDir(dir) if err != nil { @@ -91,11 +110,22 @@ func (r *Runner) runQueries(dir string) ([]Result, error) { results := make([]Result, 0, len(files)) c := compiler.New() - c.RegisterFunctions(Assertions()) + + if err := c.RegisterFunctions(Assertions()); err != nil { + return nil, err + } // read scripts for _, f := range files { - fName := filepath.Join(dir, f.Name()) + n := f.Name() + + if r.settings.Filter != nil { + if r.settings.Filter.Match([]byte(n)) != true { + continue + } + } + + fName := filepath.Join(dir, n) b, err := ioutil.ReadFile(fName) if err != nil { @@ -107,13 +137,30 @@ func (r *Runner) runQueries(dir string) ([]Result, error) { continue } - results = append(results, r.runQuery(c, fName, string(b))) + r.logger.Info().Timestamp().Str("name", fName).Msg("Running test") + + result := r.runQuery(ctx, c, fName, string(b)) + + if result.err == nil { + r.logger.Info(). + Timestamp(). + Str("file", result.name). + Msg("Test passed") + } else { + r.logger.Error(). + Timestamp(). + Err(result.err). + Str("file", result.name). + Msg("Test failed") + } + + results = append(results, result) } return results, nil } -func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result { +func (r *Runner) runQuery(ctx context.Context, c *compiler.FqlCompiler, name, script string) Result { start := time.Now() p, err := c.Compile(script) @@ -127,9 +174,10 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result { } out, err := p.Run( - context.Background(), - runtime.WithBrowser(r.settings.CDPAddress), - runtime.WithParam("server", r.settings.ServerAddress), + ctx, + runtime.WithLog(zerolog.ConsoleWriter{Out: os.Stdout}), + runtime.WithParam("static", r.settings.StaticServerAddress), + runtime.WithParam("dynamic", r.settings.DynamicServerAddress), ) duration := time.Now().Sub(start) @@ -144,7 +192,13 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result { var result string - json.Unmarshal(out, &result) + if err := json.Unmarshal(out, &result); err != nil { + return Result{ + name: name, + duration: duration, + err: err, + } + } if result == "" { return Result{ @@ -167,20 +221,9 @@ func (r *Runner) report(results []Result) Summary { for _, res := range results { if res.err != nil { - r.logger.Error(). - Timestamp(). - Err(res.err). - Str("file", res.name). - Dur("time", res.duration). - Msg("Test failed") failed++ } else { - r.logger.Info(). - Timestamp(). - Str("file", res.name). - Dur("time", res.duration). - Msg("Test passed") passed++ } diff --git a/e2e/server/server.go b/e2e/server/server.go index 80964aae..46e004fc 100644 --- a/e2e/server/server.go +++ b/e2e/server/server.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "github.com/labstack/echo" + "net/http" + "path/filepath" ) type ( @@ -22,7 +24,19 @@ func New(settings Settings) *Server { e.Debug = false e.HideBanner = true + e.Use(func(handlerFunc echo.HandlerFunc) echo.HandlerFunc { + return func(ctx echo.Context) error { + ctx.SetCookie(&http.Cookie{ + Name: "x-ferret", + Value: "e2e", + HttpOnly: false, + }) + + return handlerFunc(ctx) + } + }) e.Static("/", settings.Dir) + e.File("/", filepath.Join(settings.Dir, "index.html")) return &Server{e, settings} } diff --git a/e2e/tests/doc_cookie_del_d.fql b/e2e/tests/doc_cookie_del_d.fql new file mode 100644 index 00000000..d0a1eca7 --- /dev/null +++ b/e2e/tests/doc_cookie_del_d.fql @@ -0,0 +1,21 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, { + driver: "cdp", + cookies: [{ + name: "x-e2e", + value: "test" + }, { + name: "x-e2e-2", + value: "test2" + }] +}) + +COOKIE_DEL(doc, COOKIE_GET(doc, "x-e2e"), "x-e2e-2") + +LET cookie1 = COOKIE_GET(doc, "x-e2e") +LET cookie2 = COOKIE_GET(doc, "x-e2e-2") + +LET expected = "nonenone" +LET actual = TYPENAME(cookie1) + TYPENAME(cookie2) + +RETURN EXPECT(expected, actual) \ No newline at end of file diff --git a/e2e/tests/doc_cookie_get_d.fql b/e2e/tests/doc_cookie_get_d.fql new file mode 100644 index 00000000..8ad44e20 --- /dev/null +++ b/e2e/tests/doc_cookie_get_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, { + driver: "cdp" +}) + +LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false" +LET cookie = COOKIE_GET(doc, "x-ferret") +LET expected = "ok e2e" + +RETURN EXPECT(expected, cookiesPath + " " + cookie.value) \ No newline at end of file diff --git a/e2e/tests/doc_cookie_load_d.fql b/e2e/tests/doc_cookie_load_d.fql new file mode 100644 index 00000000..492da6e6 --- /dev/null +++ b/e2e/tests/doc_cookie_load_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, { + driver: "cdp", + cookies: [{ + name: "x-e2e", + value: "test" + }] +}) + +LET cookiesPath = LENGTH(doc.cookies) > 0 ? "ok" : "false" +LET cookie = COOKIE_GET(doc, "x-e2e") +LET expected = "ok test" + +RETURN EXPECT(expected, cookiesPath + " " + cookie.value) \ No newline at end of file diff --git a/e2e/tests/doc_cookie_set_d.fql b/e2e/tests/doc_cookie_set_d.fql new file mode 100644 index 00000000..89ddd430 --- /dev/null +++ b/e2e/tests/doc_cookie_set_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic +LET doc = DOCUMENT(@dynamic, { + driver: "cdp" +}) + +COOKIE_SET(doc, { + name: "x-e2e", + value: "test" +}) + +LET cookie = COOKIE_GET(doc, "x-e2e") +LET expected = "test" + +RETURN EXPECT(expected, cookie.value) \ No newline at end of file diff --git a/e2e/tests/doc_element_exists.fql b/e2e/tests/doc_element_exists.fql new file mode 100644 index 00000000..973e26f6 --- /dev/null +++ b/e2e/tests/doc_element_exists.fql @@ -0,0 +1,10 @@ +LET url = @static + '/overview.html' +LET doc = DOCUMENT(url) + +LET expectedP = TRUE +LET actualP = ELEMENT_EXISTS(doc, '.section-nav') + +LET expectedN = FALSE +LET actualN = ELEMENT_EXISTS(doc, '.foo-bar') + +RETURN EXPECT(expectedP + expectedN, actualP + expectedN) \ No newline at end of file diff --git a/e2e/tests/doc_element_exists_d.fql b/e2e/tests/doc_element_exists_d.fql new file mode 100644 index 00000000..e3e4142b --- /dev/null +++ b/e2e/tests/doc_element_exists_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic +LET doc = DOCUMENT(url) + +LET expectedP = TRUE +LET actualP = ELEMENT_EXISTS(doc, '.text-center') + +LET expectedN = FALSE +LET actualN = ELEMENT_EXISTS(doc, '.foo-bar') + +RETURN EXPECT(expectedP + expectedN, actualP + expectedN) \ No newline at end of file diff --git a/e2e/tests/doc_hover_d.fql b/e2e/tests/doc_hover_d.fql new file mode 100644 index 00000000..08a4810c --- /dev/null +++ b/e2e/tests/doc_hover_d.fql @@ -0,0 +1,11 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +HOVER(doc, "#hoverable-btn") +WAIT_ELEMENT(doc, "#hoverable-content") + +LET output = INNER_TEXT(doc, "#hoverable-content") + +RETURN EXPECT(output, "Lorem ipsum dolor sit amet.") \ No newline at end of file diff --git a/e2e/tests/inner_html.fql b/e2e/tests/doc_inner_html.fql similarity index 89% rename from e2e/tests/inner_html.fql rename to e2e/tests/doc_inner_html.fql index aeebfb78..b03e2cab 100644 --- a/e2e/tests/inner_html.fql +++ b/e2e/tests/doc_inner_html.fql @@ -1,4 +1,4 @@ -LET url = @server + '/bootstrap/overview.html' +LET url = @static + '/overview.html' LET doc = DOCUMENT(url) LET expected = '
  • Containers
  • Responsive breakpoints
  • Z-index
    • ' diff --git a/e2e/tests/doc_inner_html_1_arg.fql b/e2e/tests/doc_inner_html_1_arg.fql new file mode 100644 index 00000000..e7b6d866 --- /dev/null +++ b/e2e/tests/doc_inner_html_1_arg.fql @@ -0,0 +1,19 @@ +LET url = @static + '/simple.html' +LET doc = DOCUMENT(url) + +LET expected = ` + + + + Title + + +

    Hello world

    + +` +LET actual = INNER_HTML(doc) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/doc_inner_html_1_arg_d.fql b/e2e/tests/doc_inner_html_1_arg_d.fql new file mode 100644 index 00000000..cbeb0f73 --- /dev/null +++ b/e2e/tests/doc_inner_html_1_arg_d.fql @@ -0,0 +1,29 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) + +LET expected = ` + + + Ferret E2E SPA + + + + + + +

    Welcome to Ferret E2E test page!

    It has several pages for testing different possibilities of the library

    + + + + + + + + + ` +LET actual = INNER_HTML(doc) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/inner_html_all.fql b/e2e/tests/doc_inner_html_all.fql similarity index 84% rename from e2e/tests/inner_html_all.fql rename to e2e/tests/doc_inner_html_all.fql index e31402b4..d8800cb3 100644 --- a/e2e/tests/inner_html_all.fql +++ b/e2e/tests/doc_inner_html_all.fql @@ -1,4 +1,4 @@ -LET url = @server + '/bootstrap/overview.html' +LET url = @static + '/overview.html' LET doc = DOCUMENT(url) LET expected = [ diff --git a/e2e/tests/doc_inner_html_all_d.fql b/e2e/tests/doc_inner_html_all_d.fql new file mode 100644 index 00000000..6944c100 --- /dev/null +++ b/e2e/tests/doc_inner_html_all_d.fql @@ -0,0 +1,12 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#layout") + +LET expected = [ +'

    Welcome to Ferret E2E test page!

    ', +'

    It has several pages for testing different possibilities of the library

    ' +] +LET actual = INNER_HTML_ALL(doc, '#root > div > main > div > *') + +RETURN EXPECT(expected, actual) \ No newline at end of file diff --git a/e2e/tests/doc_inner_html_d.fql b/e2e/tests/doc_inner_html_d.fql new file mode 100644 index 00000000..538683da --- /dev/null +++ b/e2e/tests/doc_inner_html_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) +LET selector = '#root > div > main > div' + +WAIT_ELEMENT(doc, "#layout") + +LET expected = '

    Welcome to Ferret E2E test page!

    It has several pages for testing different possibilities of the library

    ' +LET actual = INNER_HTML(doc, selector) + +RETURN EXPECT(REGEXP_REPLACE(expected, '\s', ''), REGEXP_REPLACE(TRIM(actual), '(\n|\s)', '')) \ No newline at end of file diff --git a/e2e/tests/inner_text.fql b/e2e/tests/doc_inner_text.fql similarity index 87% rename from e2e/tests/inner_text.fql rename to e2e/tests/doc_inner_text.fql index e70b900b..837eafea 100644 --- a/e2e/tests/inner_text.fql +++ b/e2e/tests/doc_inner_text.fql @@ -1,4 +1,4 @@ -LET url = @server + '/bootstrap/overview.html' +LET url = @static + '/overview.html' LET doc = DOCUMENT(url) LET expected = "Components and options for laying out your Bootstrap project, including wrapping containers, a powerful grid system, a flexible media object, and responsive utility classes." diff --git a/e2e/tests/doc_inner_text_1_arg.fql b/e2e/tests/doc_inner_text_1_arg.fql new file mode 100644 index 00000000..0c2075ad --- /dev/null +++ b/e2e/tests/doc_inner_text_1_arg.fql @@ -0,0 +1,10 @@ +LET url = @static + '/simple.html' +LET doc = DOCUMENT(url) + +LET expected = `Title Hello world` +LET actual = INNER_TEXT(doc) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/doc_inner_text_1_arg_d.fql b/e2e/tests/doc_inner_text_1_arg_d.fql new file mode 100644 index 00000000..4de966e1 --- /dev/null +++ b/e2e/tests/doc_inner_text_1_arg_d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) + +LET expected = `Ferret E2E SPA +Ferret +Forms +Navigation +Events +Welcome to Ferret E2E test page! +It has several pages for testing different possibilities of the library +` +LET actual = INNER_TEXT(doc) + +LET r1 = '(\n|\s)' +LET r2 = '(\n|\s)' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/inner_text_all.fql b/e2e/tests/doc_inner_text_all.fql similarity index 97% rename from e2e/tests/inner_text_all.fql rename to e2e/tests/doc_inner_text_all.fql index d4cb173d..9dcbd50b 100644 --- a/e2e/tests/inner_text_all.fql +++ b/e2e/tests/doc_inner_text_all.fql @@ -1,4 +1,4 @@ -LET url = @server + '/bootstrap/grid.html' +LET url = @static + '/grid.html' LET doc = DOCUMENT(url) LET expected = [ diff --git a/e2e/tests/doc_inner_text_all_d.fql b/e2e/tests/doc_inner_text_all_d.fql new file mode 100644 index 00000000..7a638ef8 --- /dev/null +++ b/e2e/tests/doc_inner_text_all_d.fql @@ -0,0 +1,16 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) +LET selector = '#root > div > main > div > *' + +WAIT_ELEMENT(doc, "#layout") + +LET expected = [ + 'Welcome to Ferret E2E test page!', + 'It has several pages for testing different possibilities of the library' +] +LET actual = ( + FOR str IN INNER_TEXT_ALL(doc, selector) + RETURN REGEXP_REPLACE(TRIM(str), '\n', '') +) + +RETURN EXPECT(expected, actual) \ No newline at end of file diff --git a/e2e/tests/doc_inner_text_d.fql b/e2e/tests/doc_inner_text_d.fql new file mode 100644 index 00000000..4bc6977b --- /dev/null +++ b/e2e/tests/doc_inner_text_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) +LET selector = '#root > div > main > div h1' + +WAIT_ELEMENT(doc, "#layout") + +LET expected = 'Welcome to Ferret E2E test page!' +LET actual = INNER_TEXT(doc, selector) + +RETURN EXPECT(REGEXP_REPLACE(expected, '\s', ''), REGEXP_REPLACE(TRIM(actual), '(\n|\s)', '')) \ No newline at end of file diff --git a/e2e/tests/doc_input_text_d.fql b/e2e/tests/doc_input_text_d.fql new file mode 100644 index 00000000..0d2605d3 --- /dev/null +++ b/e2e/tests/doc_input_text_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET output = ELEMENT(doc, "#text_output") + +INPUT(doc, "#text_input", "foo") + +RETURN EXPECT(output.innerText, "foo") \ No newline at end of file diff --git a/e2e/tests/doc_select_multi_d.fql b/e2e/tests/doc_select_multi_d.fql new file mode 100644 index 00000000..2dbc18d5 --- /dev/null +++ b/e2e/tests/doc_select_multi_d.fql @@ -0,0 +1,9 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET output = ELEMENT(doc, "#multi_select_output") +LET result = SELECT(doc, "#multi_select_input", ["1", "2", "4"]) + +RETURN EXPECT(output.innerText, "1, 2, 4") + EXPECT(JSON_STRINGIFY(result), '["1","2","4"]') \ No newline at end of file diff --git a/e2e/tests/doc_select_single_d.fql b/e2e/tests/doc_select_single_d.fql new file mode 100644 index 00000000..6623e097 --- /dev/null +++ b/e2e/tests/doc_select_single_d.fql @@ -0,0 +1,9 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET output = ELEMENT(doc, "#select_output") +LET result = SELECT(doc, "#select_input", ["4"]) + +RETURN EXPECT(output.innerText, "4") + EXPECT(JSON_STRINGIFY(result), '["4"]') \ No newline at end of file diff --git a/e2e/tests/doc_wait_attr_all_d.fql b/e2e/tests/doc_wait_attr_all_d.fql new file mode 100644 index 00000000..5ebd1a4e --- /dev/null +++ b/e2e/tests/doc_wait_attr_all_d.fql @@ -0,0 +1,9 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +CLICK_ALL(doc, "#wait-class-btn, #wait-class-random-btn") +WAIT_ATTR_ALL(doc, "#wait-class-content, #wait-class-random-content", "class", "alert alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_attr_d.fql b/e2e/tests/doc_wait_attr_d.fql new file mode 100644 index 00000000..79df81de --- /dev/null +++ b/e2e/tests/doc_wait_attr_d.fql @@ -0,0 +1,18 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET selector = "#wait-class-btn" +LET attrName = "data-ferret-x" +LET attrVal = "foobar" + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, selector) +LET prev = el.attributes + +ATTR_SET(el, attrName, attrVal) +WAIT_ATTR(doc, selector, attrName, attrVal, 30000) +//WAIT_ATTR(el, attrName, attrVal) + +LET curr = el.attributes + +RETURN prev[attrName] == NONE && curr[attrName] == attrVal ? "" : "attributes should be updated" \ No newline at end of file diff --git a/e2e/tests/doc_wait_class_all_d.fql b/e2e/tests/doc_wait_class_all_d.fql new file mode 100644 index 00000000..2d1c4901 --- /dev/null +++ b/e2e/tests/doc_wait_class_all_d.fql @@ -0,0 +1,9 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +CLICK_ALL(doc, "#wait-class-btn, #wait-class-random-btn") +WAIT_CLASS_ALL(doc, "#wait-class-content, #wait-class-random-content", "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_class_d.fql b/e2e/tests/doc_wait_class_d.fql new file mode 100644 index 00000000..cdf433c7 --- /dev/null +++ b/e2e/tests/doc_wait_class_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +CLICK(doc, "#wait-class-btn") +WAIT_CLASS(doc, "#wait-class-content", "alert-success") + +// with random timeout +CLICK(doc, "#wait-class-random-btn") +WAIT_CLASS(doc, "#wait-class-random-content", "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_element_d.fql b/e2e/tests/doc_wait_element_d.fql new file mode 100644 index 00000000..1c61cecf --- /dev/null +++ b/e2e/tests/doc_wait_element_d.fql @@ -0,0 +1,13 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-element-content" +LET btnSelector = "#wait-element-btn" + +WAIT_ELEMENT(doc, pageSelector) + +CLICK(doc, btnSelector) + +WAIT_ELEMENT(doc, elemSelector, 10000) + +RETURN ELEMENT_EXISTS(doc, elemSelector) ? "" : "element not found" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_attr_d.fql b/e2e/tests/doc_wait_no_attr_d.fql new file mode 100644 index 00000000..56ad78e9 --- /dev/null +++ b/e2e/tests/doc_wait_no_attr_d.fql @@ -0,0 +1,16 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +CLICK(doc, "#wait-no-class-btn") +WAIT(1000) +PRINT(ATTR_GET(ELEMENT(doc, "#wait-no-class-content"), "class")) +WAIT_NO_ATTR(doc, "#wait-no-class-content", "class", "alert alert-success") + +// with random timeout +CLICK(doc, "#wait-no-class-random-btn") +WAIT_NO_ATTR(doc, "#wait-no-class-random-content", "class", "alert alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_class_all_d.fql b/e2e/tests/doc_wait_no_class_all_d.fql new file mode 100644 index 00000000..2bdb5062 --- /dev/null +++ b/e2e/tests/doc_wait_no_class_all_d.fql @@ -0,0 +1,9 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +CLICK_ALL(doc, "#wait-no-class-btn, #wait-no-class-random-btn") +WAIT_NO_CLASS_ALL(doc, "#wait-no-class-content, #wait-no-class-random-content", "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_class_d.fql b/e2e/tests/doc_wait_no_class_d.fql new file mode 100644 index 00000000..510a8a3d --- /dev/null +++ b/e2e/tests/doc_wait_no_class_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +CLICK(doc, "#wait-no-class-btn") +WAIT_NO_CLASS(doc, "#wait-no-class-content", "alert-success") + +// with random timeout +CLICK(doc, "#wait-no-class-random-btn") +WAIT_NO_CLASS(doc, "#wait-no-class-random-content", "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_element_d.fql b/e2e/tests/doc_wait_no_element_d.fql new file mode 100644 index 00000000..7ca94a6a --- /dev/null +++ b/e2e/tests/doc_wait_no_element_d.fql @@ -0,0 +1,13 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-element-content" +LET btnSelector = "#wait-no-element-btn" + +WAIT_ELEMENT(doc, pageSelector) + +CLICK(doc, btnSelector) + +WAIT_NO_ELEMENT(doc, elemSelector, 10000) + +RETURN ELEMENT_EXISTS(doc, elemSelector) ? "element should not be found" : "" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_style_all_d.fql b/e2e/tests/doc_wait_no_style_all_d.fql new file mode 100644 index 00000000..68393d9c --- /dev/null +++ b/e2e/tests/doc_wait_no_style_all_d.fql @@ -0,0 +1,30 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET selector = "#wait-class-btn, #wait-class-random-btn" + +WAIT_ELEMENT(doc, "#page-events") + +LET n = ( + FOR el IN ELEMENTS(doc, selector) + ATTR_SET(el, "style", "color: black") + + RETURN NONE +) + +WAIT_STYLE_ALL(doc, selector, "color", "black", 10000) + +LET n2 = ( + FOR el IN ELEMENTS(doc, selector) + ATTR_SET(el, "style", "color: red") + + RETURN NONE +) + +WAIT_NO_STYLE_ALL(doc, selector, "color", "black", 10000) + +LET results = ( + FOR el IN ELEMENTS(doc, selector) + RETURN el.style.color +) + +RETURN CONCAT(results) == "redred" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/doc_wait_no_style_d.fql b/e2e/tests/doc_wait_no_style_d.fql new file mode 100644 index 00000000..c507f930 --- /dev/null +++ b/e2e/tests/doc_wait_no_style_d.fql @@ -0,0 +1,19 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET selector = "#wait-class-btn" + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, selector) + +ATTR_SET(el, "style", "width: 100%") +WAIT_STYLE(doc, selector, "width", "100%") + +LET prev = el.style + +ATTR_SET(el, "style", "width: 50%") +WAIT_NO_STYLE(doc, selector, "width", "100%") + +LET curr = el.style + +RETURN prev.width == "100%" && curr.width == "50%" ? "" : "style should be changed" \ No newline at end of file diff --git a/e2e/tests/doc_wait_style_all_d.fql b/e2e/tests/doc_wait_style_all_d.fql new file mode 100644 index 00000000..15f1eb07 --- /dev/null +++ b/e2e/tests/doc_wait_style_all_d.fql @@ -0,0 +1,21 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET selector = "#wait-class-btn, #wait-class-random-btn" + +WAIT_ELEMENT(doc, "#page-events") + +LET n = ( + FOR el IN ELEMENTS(doc, selector) + ATTR_SET(el, "style", "color: black") + + RETURN NONE +) + +WAIT_STYLE_ALL(doc, selector, "color", "black", 10000) + +LET results = ( + FOR el IN ELEMENTS(doc, selector) + RETURN el.style.color +) + +RETURN CONCAT(results) == "blackblack" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/doc_wait_style_d.fql b/e2e/tests/doc_wait_style_d.fql new file mode 100644 index 00000000..1807de03 --- /dev/null +++ b/e2e/tests/doc_wait_style_d.fql @@ -0,0 +1,15 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET selector = "#wait-class-btn" + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, selector) +LET prev = el.style + +ATTR_SET(el, "style", "width: 100%") +WAIT_STYLE(doc, selector, "width", "100%") + +LET curr = el.style + +RETURN prev.width == NONE && curr.width == "100%" ? "" : "style should be updated" \ No newline at end of file diff --git a/e2e/tests/el_attrs.fql b/e2e/tests/el_attrs.fql new file mode 100644 index 00000000..6016a085 --- /dev/null +++ b/e2e/tests/el_attrs.fql @@ -0,0 +1,13 @@ +LET url = @static + '/overview.html' +LET doc = DOCUMENT(url) + +LET el = ELEMENT(doc, "body > header > a") +LET attrs = [ + el.attributes.class, + el.attributes.href +] + +LET expected = '["navbar-brand mr-0 mr-md-2","http://getbootstrap.com/"]' +LET actual = TO_STRING(attrs) + +RETURN EXPECT(expected, actual) \ No newline at end of file diff --git a/e2e/tests/el_attrs_d.fql b/e2e/tests/el_attrs_d.fql new file mode 100644 index 00000000..11c137ce --- /dev/null +++ b/e2e/tests/el_attrs_d.fql @@ -0,0 +1,13 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, { driver: "cdp" }) + +LET el = ELEMENT(doc, "#index") +LET attrs = [ + el.attributes.class, + el.attributes["data-type"] +] + +LET expected = '["jumbotron","page"]' +LET actual = TO_STRING(attrs) + +RETURN EXPECT(expected, actual) \ No newline at end of file diff --git a/e2e/tests/el_attrs_get.d.fql b/e2e/tests/el_attrs_get.d.fql new file mode 100644 index 00000000..cc04d98a --- /dev/null +++ b/e2e/tests/el_attrs_get.d.fql @@ -0,0 +1,11 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) +LET attrs = ATTR_GET(el, "style") + +RETURN EXPECT("display: block;", attrs.style) \ No newline at end of file diff --git a/e2e/tests/el_attrs_remove.d.fql b/e2e/tests/el_attrs_remove.d.fql new file mode 100644 index 00000000..b81e92a9 --- /dev/null +++ b/e2e/tests/el_attrs_remove.d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) +LET el = ELEMENT(doc, elemSelector) + +LET prev = el.attributes.style + +ATTR_REMOVE(el, "style") + +WAIT(1000) + +LET curr = el.attributes.style + +RETURN prev == "display: block;" && curr == NONE ? "" : "expected attribute to be removed" \ No newline at end of file diff --git a/e2e/tests/el_attrs_set.d.fql b/e2e/tests/el_attrs_set.d.fql new file mode 100644 index 00000000..766f1cdf --- /dev/null +++ b/e2e/tests/el_attrs_set.d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) +LET prev = el.style + +ATTR_SET(el, "style", "color: black;") + +WAIT(1000) + +LET curr = el.style + +RETURN curr.color == "black" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/el_attrs_set_bulk.d.fql b/e2e/tests/el_attrs_set_bulk.d.fql new file mode 100644 index 00000000..b4cab363 --- /dev/null +++ b/e2e/tests/el_attrs_set_bulk.d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) + +ATTR_SET(el, { style: "color: black;", "data-ferret-x": "test" }) + +WAIT(1000) + +RETURN el.style.color == "black" && el.attributes["data-ferret-x"] == "test" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/el_element_exists.fql b/e2e/tests/el_element_exists.fql new file mode 100644 index 00000000..6225f672 --- /dev/null +++ b/e2e/tests/el_element_exists.fql @@ -0,0 +1,12 @@ +LET url = @static + '/value.html' +LET doc = DOCUMENT(url) + +LET el = ELEMENT(doc, "#listings_table") + +LET expectedP = TRUE +LET actualP = ELEMENT_EXISTS(el, '.odd') + +LET expectedN = FALSE +LET actualN = ELEMENT_EXISTS(el, '.foo-bar') + +RETURN EXPECT(expectedP + expectedN, actualP + expectedN) \ No newline at end of file diff --git a/e2e/tests/el_element_exists_d.fql b/e2e/tests/el_element_exists_d.fql new file mode 100644 index 00000000..ac2be8d7 --- /dev/null +++ b/e2e/tests/el_element_exists_d.fql @@ -0,0 +1,12 @@ +LET url = @dynamic +LET doc = DOCUMENT(url) + +LET el = ELEMENT(doc, "#root") + +LET expectedP = TRUE +LET actualP = ELEMENT_EXISTS(el, '.jumbotron') + +LET expectedN = FALSE +LET actualN = ELEMENT_EXISTS(el, '.foo-bar') + +RETURN EXPECT(expectedP + expectedN, actualP + expectedN) \ No newline at end of file diff --git a/e2e/tests/el_hover_d.fql b/e2e/tests/el_hover_d.fql new file mode 100644 index 00000000..f9b836e3 --- /dev/null +++ b/e2e/tests/el_hover_d.fql @@ -0,0 +1,13 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +LET input = ELEMENT(doc, "#hoverable-btn") + +HOVER(input) +WAIT_ELEMENT(doc, "#hoverable-content") + +LET output = ELEMENT(doc, "#hoverable-content") + +RETURN EXPECT(output.innerText, "Lorem ipsum dolor sit amet.") \ No newline at end of file diff --git a/e2e/tests/el_inner_html_1_arg.fql b/e2e/tests/el_inner_html_1_arg.fql new file mode 100644 index 00000000..fe962f24 --- /dev/null +++ b/e2e/tests/el_inner_html_1_arg.fql @@ -0,0 +1,11 @@ +LET url = @static + '/simple.html' +LET doc = DOCUMENT(url) +LET el = ELEMENT(doc, "body") + +LET expected = `

    Hello world

    ` +LET actual = INNER_HTML(el) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/el_inner_html_1_arg_d.fql b/e2e/tests/el_inner_html_1_arg_d.fql new file mode 100644 index 00000000..d10ba8d2 --- /dev/null +++ b/e2e/tests/el_inner_html_1_arg_d.fql @@ -0,0 +1,11 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) +LET el = ELEMENT(doc, "#root") + +LET expected = `

    Welcome to Ferret E2E test page!

    It has several pages for testing different possibilities of the library

    ` +LET actual = INNER_HTML(el) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/el_inner_text_1_arg.fql b/e2e/tests/el_inner_text_1_arg.fql new file mode 100644 index 00000000..6deeda2e --- /dev/null +++ b/e2e/tests/el_inner_text_1_arg.fql @@ -0,0 +1,11 @@ +LET url = @static + '/simple.html' +LET doc = DOCUMENT(url) +LET el = ELEMENT(doc, "body") + +LET expected = `Hello world` +LET actual = INNER_TEXT(el) + +LET r1 = '(\s|\")' +LET r2 = '(\n|\s|\")' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/el_inner_text_1_arg_d.fql b/e2e/tests/el_inner_text_1_arg_d.fql new file mode 100644 index 00000000..ff1b5da7 --- /dev/null +++ b/e2e/tests/el_inner_text_1_arg_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) +LET el = ELEMENT(doc, ".jumbotron") + +LET expected = ` +Welcome to Ferret E2E test page! +It has several pages for testing different possibilities of the library +` +LET actual = INNER_TEXT(el) + +LET r1 = '(\n|\s)' +LET r2 = '(\n|\s)' + +RETURN EXPECT(REGEXP_REPLACE(expected, r1, ''), REGEXP_REPLACE(TRIM(actual), r2, '')) \ No newline at end of file diff --git a/e2e/tests/el_input_text_d.fql b/e2e/tests/el_input_text_d.fql new file mode 100644 index 00000000..79301ab8 --- /dev/null +++ b/e2e/tests/el_input_text_d.fql @@ -0,0 +1,11 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET input = ELEMENT(doc, "#text_input") +LET output = ELEMENT(doc, "#text_output") + +INPUT(input, "foo") + +RETURN EXPECT(output.innerText, "foo") \ No newline at end of file diff --git a/e2e/tests/el_select_multi_d.fql b/e2e/tests/el_select_multi_d.fql new file mode 100644 index 00000000..d0332498 --- /dev/null +++ b/e2e/tests/el_select_multi_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET input = ELEMENT(doc, "#multi_select_input") +LET output = ELEMENT(doc, "#multi_select_output") +LET result = SELECT(input, ["1", "2", "4"]) + +RETURN EXPECT(output.innerText, "1, 2, 4") + EXPECT(JSON_STRINGIFY(result), '["1","2","4"]') \ No newline at end of file diff --git a/e2e/tests/el_select_single_d.fql b/e2e/tests/el_select_single_d.fql new file mode 100644 index 00000000..8964511e --- /dev/null +++ b/e2e/tests/el_select_single_d.fql @@ -0,0 +1,10 @@ +LET url = @dynamic + "?redirect=/forms" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "form") + +LET input = ELEMENT(doc, "#select_input") +LET output = ELEMENT(doc, "#select_output") +LET result = SELECT(input, ["4"]) + +RETURN EXPECT(output.innerText, "4") + EXPECT(JSON_STRINGIFY(result), '["4"]') \ No newline at end of file diff --git a/e2e/tests/el_style_get.d.fql b/e2e/tests/el_style_get.d.fql new file mode 100644 index 00000000..77d00cbd --- /dev/null +++ b/e2e/tests/el_style_get.d.fql @@ -0,0 +1,11 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) +LET val = STYLE_GET(el, "display") + +RETURN val.display == "block" ? "" : "could not get style values" \ No newline at end of file diff --git a/e2e/tests/el_style_remove.d.fql b/e2e/tests/el_style_remove.d.fql new file mode 100644 index 00000000..2ffc66af --- /dev/null +++ b/e2e/tests/el_style_remove.d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) +LET el = ELEMENT(doc, elemSelector) + +LET prev = el.style + +STYLE_REMOVE(el, "display") + +WAIT(1000) + +LET curr = el.style + +RETURN prev.display == "block" && curr.display == NONE ? "" : "expected style to be removed" \ No newline at end of file diff --git a/e2e/tests/el_style_set.d.fql b/e2e/tests/el_style_set.d.fql new file mode 100644 index 00000000..e7770b51 --- /dev/null +++ b/e2e/tests/el_style_set.d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) +LET prev = el.style + +STYLE_SET(el, "color", "black") + +WAIT(1000) + +LET curr = el.style + +RETURN curr.color == "black" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/el_style_set_bulk.d.fql b/e2e/tests/el_style_set_bulk.d.fql new file mode 100644 index 00000000..5b876553 --- /dev/null +++ b/e2e/tests/el_style_set_bulk.d.fql @@ -0,0 +1,17 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) +LET pageSelector = "#page-events" +LET elemSelector = "#wait-no-style-content" + +WAIT_ELEMENT(doc, pageSelector) + +LET el = ELEMENT(doc, elemSelector) +LET prev = el.style + +STYLE_SET(el, { color: "black", "min-width": "100px", "background-color": "#11111" }) + +WAIT(1000) + +LET curr = el.style + +RETURN curr.color == "black" && curr["min-width"] == "100px" && curr["background-color"] == "#11111" ? "" : "styles should be updated" \ No newline at end of file diff --git a/e2e/tests/el_value.fql b/e2e/tests/el_value.fql new file mode 100644 index 00000000..75f3c406 --- /dev/null +++ b/e2e/tests/el_value.fql @@ -0,0 +1,12 @@ +LET url = @static + '/value.html' +LET doc = DOCUMENT(url) + +LET expected = ["068728","068728","816410","52024413","698690","210583","049700","826394","354369","135911","700285","557242","278832","357701","313034","959368","703500","842750","777175","378061","072489","383005","843393","59912263","464535","229710","230550","767964","758862","944384","025449","010245","844935","038760","013450","124139","211145","758761","448667","488966"] + +LET actual = ( + FOR tr IN ELEMENTS(doc, '#listings_table > tbody > tr') + LET elem = ELEMENT(tr, 'td > input') + RETURN elem.value +) + +RETURN EXPECT(actual, expected) \ No newline at end of file diff --git a/e2e/tests/el_value_d.fql b/e2e/tests/el_value_d.fql new file mode 100644 index 00000000..c5463313 --- /dev/null +++ b/e2e/tests/el_value_d.fql @@ -0,0 +1,12 @@ +LET url = @static + '/value.html' +LET doc = DOCUMENT(url, true) + +LET expected = ["068728","068728","816410","52024413","698690","210583","049700","826394","354369","135911","700285","557242","278832","357701","313034","959368","703500","842750","777175","378061","072489","383005","843393","59912263","464535","229710","230550","767964","758862","944384","025449","010245","844935","038760","013450","124139","211145","758761","448667","488966"] + +LET actual = ( + FOR tr IN ELEMENTS(doc, '#listings_table > tbody > tr') + LET elem = ELEMENT(tr, 'td > input') + RETURN elem.value +) + +RETURN EXPECT(actual, expected) \ No newline at end of file diff --git a/e2e/tests/el_wait_attr_2_d.fql b/e2e/tests/el_wait_attr_2_d.fql new file mode 100644 index 00000000..139786e6 --- /dev/null +++ b/e2e/tests/el_wait_attr_2_d.fql @@ -0,0 +1,14 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, "#wait-class-content") + +ATTR_SET(el, "data-test", "test") +WAIT_ATTR(el, "data-test", "test") + +ATTR_REMOVE(el, "class") +WAIT_ATTR(el, "class", NONE) + +RETURN el.attributes.class == NONE ? "" : "attribute should be removed" \ No newline at end of file diff --git a/e2e/tests/el_wait_attr_d.fql b/e2e/tests/el_wait_attr_d.fql new file mode 100644 index 00000000..4ca63801 --- /dev/null +++ b/e2e/tests/el_wait_attr_d.fql @@ -0,0 +1,20 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +LET b1 = ELEMENT(doc, "#wait-class-btn") +LET c1 = ELEMENT(doc, "#wait-class-content") + +CLICK(b1) +WAIT_ATTR(c1, "class", "alert alert-success") + +// with random timeout +LET b2 = ELEMENT(doc, "#wait-class-random-btn") +LET c2 = ELEMENT(doc, "#wait-class-random-content") + +CLICK(b2) +WAIT_ATTR(c2, "class", "alert alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/el_wait_class_d.fql b/e2e/tests/el_wait_class_d.fql new file mode 100644 index 00000000..a0d93c79 --- /dev/null +++ b/e2e/tests/el_wait_class_d.fql @@ -0,0 +1,20 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +LET b1 = ELEMENT(doc, "#wait-class-btn") +LET c1 = ELEMENT(doc, "#wait-class-content") + +CLICK(b1) +WAIT_CLASS(c1, "alert-success") + +// with random timeout +LET b2 = ELEMENT(doc, "#wait-class-random-btn") +LET c2 = ELEMENT(doc, "#wait-class-random-content") + +CLICK(b2) +WAIT_CLASS(c2, "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/el_wait_no_attr_d.fql b/e2e/tests/el_wait_no_attr_d.fql new file mode 100644 index 00000000..b467296e --- /dev/null +++ b/e2e/tests/el_wait_no_attr_d.fql @@ -0,0 +1,20 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +LET b1 = ELEMENT(doc, "#wait-no-class-btn") +LET c1 = ELEMENT(doc, "#wait-no-class-content") + +CLICK(b1) +WAIT_NO_ATTR(c1, "class", "alert alert-success") + +// with random timeout +LET b2 = ELEMENT(doc, "#wait-no-class-random-btn") +LET c2 = ELEMENT(doc, "#wait-no-class-random-content") + +CLICK(b2) +WAIT_NO_ATTR(c2, "class", "alert alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/el_wait_no_class_d.fql b/e2e/tests/el_wait_no_class_d.fql new file mode 100644 index 00000000..b15136e0 --- /dev/null +++ b/e2e/tests/el_wait_no_class_d.fql @@ -0,0 +1,20 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +// with fixed timeout +LET b1 = ELEMENT(doc, "#wait-no-class-btn") +LET c1 = ELEMENT(doc, "#wait-no-class-content") + +CLICK(b1) +WAIT_NO_CLASS(c1, "alert-success") + +// with random timeout +LET b2 = ELEMENT(doc, "#wait-no-class-random-btn") +LET c2 = ELEMENT(doc, "#wait-no-class-random-content") + +CLICK(b2) +WAIT_NO_CLASS(c2, "alert-success", 10000) + +RETURN "" \ No newline at end of file diff --git a/e2e/tests/el_wait_no_style_d.fql b/e2e/tests/el_wait_no_style_d.fql new file mode 100644 index 00000000..26a249a1 --- /dev/null +++ b/e2e/tests/el_wait_no_style_d.fql @@ -0,0 +1,18 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, "#wait-class-content") + +ATTR_SET(el, "style", "color: black") +WAIT_STYLE(el, "color", "black") + +LET prev = el.style + +ATTR_SET(el, "style", "color: red") +WAIT_NO_STYLE(el, "color", "black") + +LET curr = el.style + +RETURN prev.color == "black" && curr.color == "red" ? "" : "style should be changed" \ No newline at end of file diff --git a/e2e/tests/el_wait_style_d.fql b/e2e/tests/el_wait_style_d.fql new file mode 100644 index 00000000..073b6263 --- /dev/null +++ b/e2e/tests/el_wait_style_d.fql @@ -0,0 +1,18 @@ +LET url = @dynamic + "?redirect=/events" +LET doc = DOCUMENT(url, true) + +WAIT_ELEMENT(doc, "#page-events") + +LET el = ELEMENT(doc, "#wait-class-content") + +ATTR_SET(el, "style", "color: black") +WAIT_STYLE(el, "color", "black") + +LET prev = el.style + +ATTR_REMOVE(el, "style") +WAIT_STYLE(el, "color", NONE) + +LET curr = el.style + +RETURN prev.color == "black" && curr.color == NONE ? "" : "style should be removed" \ No newline at end of file diff --git a/e2e/tests/inner_html_dynamic.fql b/e2e/tests/inner_html_dynamic.fql deleted file mode 100644 index c1041448..00000000 --- a/e2e/tests/inner_html_dynamic.fql +++ /dev/null @@ -1,10 +0,0 @@ -// LET url = @server + '/bootstrap/overview.html' -// LET doc = DOCUMENT(url, true) -// LET selector = '.section-nav' - -// LET expected = '
  • Containers
  • Responsive breakpoints
  • Z-index
    • ' -// LET actual = INNER_HTML(doc, selector) - -// RETURN EXPECT(REGEXP_REPLACE(expected, '\s', ''), REGEXP_REPLACE(TRIM(actual), '(\n|\s)', '')) - -RETURN "" \ No newline at end of file diff --git a/e2e/tests/inner_text_dynamic.fql b/e2e/tests/inner_text_dynamic.fql deleted file mode 100644 index a2b90393..00000000 --- a/e2e/tests/inner_text_dynamic.fql +++ /dev/null @@ -1,9 +0,0 @@ -// LET url = @server + '/bootstrap/overview.html' -// LET doc = DOCUMENT(url, true) -// LET selector = 'body > div > div > main > p.bd-lead' - -// LET expected = "Components and options for laying out your Bootstrap project, including wrapping containers, a powerful grid system, a flexible media object, and responsive utility classes." -// LET actual = INNER_TEXT(doc, selector) - -// RETURN EXPECT(expected, actual) -RETURN "" \ No newline at end of file diff --git a/e2e/tests/page-load.fql b/e2e/tests/page_load.fql similarity index 52% rename from e2e/tests/page-load.fql rename to e2e/tests/page_load.fql index f7fdab18..302887e3 100644 --- a/e2e/tests/page-load.fql +++ b/e2e/tests/page_load.fql @@ -1,4 +1,4 @@ -LET url = @server + '/bootstrap/overview.html' +LET url = @static + '/overview.html' LET doc = DOCUMENT(url) RETURN EXPECT(doc.url, url) \ No newline at end of file diff --git a/e2e/tests/page_load_d.fql b/e2e/tests/page_load_d.fql new file mode 100644 index 00000000..f4fedabc --- /dev/null +++ b/e2e/tests/page_load_d.fql @@ -0,0 +1,4 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, true) + +RETURN EXPECT(doc.url, url) \ No newline at end of file diff --git a/e2e/tests/page_load_params_d.fql b/e2e/tests/page_load_params_d.fql new file mode 100644 index 00000000..4bb1fc08 --- /dev/null +++ b/e2e/tests/page_load_params_d.fql @@ -0,0 +1,6 @@ +LET url = @dynamic +LET doc = DOCUMENT(url, { + dynamic: true +}) + +RETURN EXPECT(doc.url, url) \ No newline at end of file diff --git a/examples/cookies.go b/examples/cookies.go new file mode 100644 index 00000000..c1f4b85c --- /dev/null +++ b/examples/cookies.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" +) + +func run(q string) ([]byte, error) { + comp := compiler.New() + program := comp.MustCompile(q) + + // create a root context + ctx := context.Background() + + // we inform the driver to keep cookies between queries + ctx = drivers.WithContext( + ctx, + cdp.NewDriver(cdp.WithKeepCookies()), + drivers.AsDefault(), + ) + + return program.Run(ctx) +} diff --git a/examples/crawler.fql b/examples/crawler.fql index 959e7049..22322787 100644 --- a/examples/crawler.fql +++ b/examples/crawler.fql @@ -1,4 +1,6 @@ -LET doc = DOCUMENT('https://www.theverge.com/tech', true) +LET doc = DOCUMENT('https://www.theverge.com/tech', { + driver: "cdp" +}) WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000) LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper') LET links = ( diff --git a/examples/embedded.go b/examples/embedded.go index 8419a52d..9712972e 100644 --- a/examples/embedded.go +++ b/examples/embedded.go @@ -4,8 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/MontFerret/ferret/pkg/compiler" "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp" + "github.com/MontFerret/ferret/pkg/drivers/http" ) type Topic struct { @@ -52,7 +56,17 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return nil, err } - out, err := program.Run(context.Background()) + // create a root context + ctx := context.Background() + + // enable HTML drivers + // by default, Ferret Runtime does not know about any HTML drivers + // all HTML manipulations are done via functions from standard library + // that assume that at least one driver is available + ctx = drivers.WithContext(ctx, cdp.NewDriver()) + ctx = drivers.WithContext(ctx, http.NewDriver(), drivers.AsDefault()) + + out, err := program.Run(ctx) if err != nil { return nil, err diff --git a/examples/extensible.go b/examples/extensible.go index c5b5b186..ea27b066 100644 --- a/examples/extensible.go +++ b/examples/extensible.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "os" + "strings" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" - "os" ) func main() { @@ -27,7 +29,7 @@ func getStrings() ([]string, error) { // function implements is a type of a function that ferret supports as a runtime function transform := func(ctx context.Context, args ...core.Value) (core.Value, error) { // it's just a helper function which helps to validate a number of passed args - err := core.ValidateArgs(args, 1) + err := core.ValidateArgs(args, 1, 1) if err != nil { // it's recommended to return built-in None type, instead of nil @@ -44,7 +46,7 @@ func getStrings() ([]string, error) { // cast to built-in string type str := args[0].(values.String) - return str.Concat(values.NewString("_ferret")).ToUpper(), nil + return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil } query := ` @@ -54,7 +56,10 @@ func getStrings() ([]string, error) { ` comp := compiler.New() - comp.RegisterFunction("transform", transform) + + if err := comp.RegisterFunction("transform", transform); err != nil { + return nil, err + } program, err := comp.Compile(query) diff --git a/examples/navigate_forward_2.fql b/examples/navigate_forward_2.fql index b6fe901a..dedfea51 100644 --- a/examples/navigate_forward_2.fql +++ b/examples/navigate_forward_2.fql @@ -1,9 +1,9 @@ LET doc = DOCUMENT("https://github.com/", true) NAVIGATE(doc, "https://github.com/features") -NAVIGATE(doc, "https://github.com/business") +NAVIGATE(doc, "https://github.com/enterprise") NAVIGATE(doc, "https://github.com/marketplace") NAVIGATE_BACK(doc, 3) NAVIGATE_FORWARD(doc, 2) -RETURN doc.url == "https://github.com/business" +RETURN doc.url == "https://github.com/enterprise" diff --git a/examples/pagination.fql b/examples/pagination.fql index cb8f10f3..a6ecbd7e 100644 --- a/examples/pagination.fql +++ b/examples/pagination.fql @@ -5,31 +5,35 @@ CLICK(amazon, '.nav-search-submit input[type="submit"]') WAIT_NAVIGATION(amazon) LET resultListSelector = '#s-results-list-atf' -LET resultItemSelector = '.s-result-item' +LET resultItemSelector = '.s-result-item.celwidget' LET nextBtnSelector = '#pagnNextLink' -LET vendorSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div.a-row.a-spacing-small > div:nth-child(2) > span:nth-child(2)' -LET priceSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div:nth-child(4) > div.a-column.a-span7 > div:nth-child(1) > div:nth-child(3) > a > span.a-offscreen' -LET altPriceSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div:nth-child(2) > div.a-column.a-span7 > div:nth-child(1) > div:nth-child(3) > a > span.a-offscreen' +LET vendorSelector1 = 'div > div:nth-child(3) > div:nth-child(2) > span:nth-child(2)' +LET vendorSelector2 = 'div > div:nth-child(5) > div:nth-child(2) > span:nth-child(2)' +LET priceWholeSelector = 'span.sx-price-whole' +LET priceFracSelector = 'sup.sx-price-fractional' +LET pages = TO_INT(INNER_TEXT(amazon, '#pagn > span.pagnDisabled')) LET result = ( - FOR pageNum IN 1..@pages + FOR pageNum IN 1..pages LET clicked = pageNum == 1 ? false : CLICK(amazon, nextBtnSelector) LET wait = clicked ? WAIT_NAVIGATION(amazon) : false LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false LET items = ( FOR el IN ELEMENTS(amazon, resultItemSelector) - LET priceTxtMain = INNER_TEXT(el, priceSelector) - LET priceTxt = priceTxtMain != "" ? priceTxtMain : INNER_TEXT(el, altPriceSelector) + LET priceWholeTxt = INNER_TEXT(el, priceWholeSelector) + LET priceFracTxt = INNER_TEXT(el, priceFracSelector) + LET price = TO_FLOAT(priceWholeTxt + "." + priceFracTxt) + LET vendor = ELEMENT_EXISTS(el, vendorSelector1) ? INNER_TEXT(el, vendorSelector1) : INNER_TEXT(el, vendorSelector2) RETURN { title: INNER_TEXT(el, 'h2'), - vendor: INNER_TEXT(el, vendorSelector), - price: TO_FLOAT(SUBSTITUTE(priceTxt, "$", "")) + vendor, + price } ) RETURN items ) -RETURN FLATTEN(result) \ No newline at end of file +RETURN FLATTEN(result) diff --git a/examples/pagination_uncontrolled.fql b/examples/pagination_uncontrolled.fql new file mode 100644 index 00000000..9293a97c --- /dev/null +++ b/examples/pagination_uncontrolled.fql @@ -0,0 +1,36 @@ +LET amazon = DOCUMENT('https://www.amazon.com/', true) + +INPUT(amazon, '#twotabsearchtextbox', @criteria) +CLICK(amazon, '.nav-search-submit input[type="submit"]') +WAIT_NAVIGATION(amazon) + +LET resultListSelector = '#s-results-list-atf' +LET resultItemSelector = '.s-result-item' +LET nextBtnSelector = '#pagnNextLink' +LET vendorSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div.a-row.a-spacing-small > div:nth-child(2) > span:nth-child(2)' +LET priceSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div:nth-child(4) > div.a-column.a-span7 > div:nth-child(1) > div:nth-child(3) > a > span.a-offscreen' +LET altPriceSelector = 'div > div > div > div.a-fixed-left-grid-col.a-col-right > div:nth-child(2) > div.a-column.a-span7 > div:nth-child(1) > div:nth-child(3) > a > span.a-offscreen' + +LET result = ( + FOR pageNum IN PAGINATION(amazon, nextBtnSelector) + LIMIT @limit + + LET wait = pageNum > 0 ? WAIT_NAVIGATION(amazon) : false + LET waitSelector = wait ? WAIT_ELEMENT(amazon, resultListSelector) : false + + LET items = ( + FOR el IN ELEMENTS(amazon, resultItemSelector) + LET priceTxtMain = INNER_TEXT(el, priceSelector) + LET priceTxt = priceTxtMain != "" ? priceTxtMain : INNER_TEXT(el, altPriceSelector) + + RETURN { + title: INNER_TEXT(el, 'h2'), + vendor: INNER_TEXT(el, vendorSelector), + price: TO_FLOAT(SUBSTITUTE(priceTxt, "$", "")) + } + ) + + RETURN items +) + +RETURN FLATTEN(result) \ No newline at end of file diff --git a/examples/wait_class.fql b/examples/wait_class.fql index f179989c..6deef221 100644 --- a/examples/wait_class.fql +++ b/examples/wait_class.fql @@ -1,5 +1,4 @@ LET doc = DOCUMENT("http://getbootstrap.com/docs/4.1/components/collapse/", true) - LET el = ELEMENT(doc, "#collapseTwo") CLICK(doc, "#headingTwo > h5 > button") diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..61f90f58 --- /dev/null +++ b/go.mod @@ -0,0 +1,76 @@ +module github.com/MontFerret/ferret + +go 1.12 + +require ( + github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect + github.com/PuerkitoBio/goquery v1.5.0 + github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect + github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47 + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e + github.com/coreos/etcd v3.3.12+incompatible // indirect + github.com/corpix/uarand v0.0.0 + github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9 + github.com/fatih/color v1.7.0 // indirect + github.com/go-critic/go-critic v0.3.4 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/go-toolsmith/astcast v1.0.0 // indirect + github.com/go-toolsmith/astcopy v1.0.0 // indirect + github.com/go-toolsmith/astfmt v1.0.0 // indirect + github.com/go-toolsmith/astp v1.0.0 // indirect + github.com/go-toolsmith/pkgload v1.0.0 // indirect + github.com/go-toolsmith/typep v1.0.0 // indirect + github.com/gofrs/uuid v3.2.0+incompatible + github.com/gogo/protobuf v1.2.1 // indirect + github.com/golang/mock v1.2.0 // indirect + github.com/golang/protobuf v1.3.1 // indirect + github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect + github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect + github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect + github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect + github.com/golangci/golangci-lint v1.15.0 // indirect + github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect + github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect + github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect + github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect + github.com/gorilla/css v1.0.0 + github.com/gorilla/websocket v1.4.0 // indirect + github.com/kisielk/errcheck v1.2.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/kr/pty v1.1.4 // indirect + github.com/labstack/echo v3.3.10+incompatible + github.com/labstack/gommon v0.2.8 // indirect + github.com/mafredri/cdp v0.23.0 + github.com/mattn/go-colorable v0.1.1 // indirect + github.com/mattn/go-isatty v0.0.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mozilla/tls-observatory v0.0.0-20190313211306-43961c0c7a1f // indirect + github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/rogpeppe/go-internal v1.2.2 // indirect + github.com/rs/zerolog v1.13.0 + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 + github.com/shirou/gopsutil v2.18.12+incompatible // indirect + github.com/shurcooL/go v0.0.0-20190121191506-3fef8c783dec // indirect + github.com/sirupsen/logrus v1.4.0 // indirect + github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac // indirect + github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff + github.com/spf13/afero v1.2.2 // indirect + github.com/spf13/cobra v0.0.3 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/viper v1.3.2 // indirect + github.com/stretchr/testify v1.3.0 // indirect + github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect + github.com/valyala/fasttemplate v1.0.1 // indirect + golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect + golang.org/x/net v0.0.0-20190328230028-74de082e2cca + golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 + golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect + golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 // indirect + mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect + sourcegraph.com/sqs/pbtypes v1.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..b0136ea7 --- /dev/null +++ b/go.sum @@ -0,0 +1,254 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= +github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47 h1:Lp5nUoQzppfVmfZadpzAytNyb5IMtxyOJLzoQS5dExg= +github.com/antlr/antlr4 v0.0.0-20190325153624-837aa60e2c47/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/corpix/uarand v0.0.0 h1:mNbzro1GwUcZ1hmO2rWXytkR3JBxNxxctzjyuhO+Aig= +github.com/corpix/uarand v0.0.0/go.mod h1:JSm890tOkDN+M1jqN8pUGDKnzJrsVbJwSMHBY4zwz7M= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9 h1:aSaTVlEXc2QKl4fzXU1tMYCjlrSc2mA4DZtiVfckQHo= +github.com/derekparker/trie v0.0.0-20190322172448-1ce4922c7ad9/go.mod h1:D6ICZm05D9VN1n/8iOtBxLpXtoGp6HDFUJ1RNVieOSE= +github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead/go.mod h1:3MzXZKJdeXqdU9cj+rvZdNiN7SZ8V9OjybF8loZDmHU= +github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc= +github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= +github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.15.0 h1:Cvf5J2U5C5kAHSXxyLRJ38/OqdJbiU7SAJRYvWgO/b0= +github.com/golangci/golangci-lint v1.15.0/go.mod h1:iEsyA2h6yMxPzFAlb/Q9UuXBrXIDtXkbUoukuqUAX/8= +github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= +github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo= +github.com/golangci/govet v0.0.0-20180818181408-44ddbe260190/go.mod h1:pPwb+AK755h3/r73avHz5bEN6sa51/2HEZlLaV53hCo= +github.com/golangci/ineffassign v0.0.0-20180808204949-2ee8f2867dde/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20180610141402-4bf9709227d1/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= +github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0= +github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mafredri/cdp v0.22.0 h1:BV17j8hXLDWczo2SZIAFuOjMpQMIOq5DOcd9sgB2hv0= +github.com/mafredri/cdp v0.22.0/go.mod h1:hgdiA0yp1uqhSaDOHJWPgXpMbh+LAfUdD9vbN2AM8gE= +github.com/mafredri/cdp v0.23.0 h1:Qrzms9wVwOpJdbF3ooxZcBTnEqjiqHQkT8kCyTtjoPg= +github.com/mafredri/cdp v0.23.0/go.mod h1:hgdiA0yp1uqhSaDOHJWPgXpMbh+LAfUdD9vbN2AM8gE= +github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/mozilla/tls-observatory v0.0.0-20190313211306-43961c0c7a1f/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k= +github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go v0.0.0-20190121191506-3fef8c783dec/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac h1:wbW+Bybf9pXxnCFAOWZTqkRjAc7rAIwo2e1ArUhiHxg= +github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2 h1:17UhVDrPb40BH5k6cyeb2V/7QlBNdo/a0+r0dtK+Utw= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-379209517ffe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= +mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o= +sourcegraph.com/sourcegraph/go-diff v0.5.1-0.20190210232911-dee78e514455/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= +sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4= diff --git a/main.go b/main.go index 4f66d87b..1b6aa08e 100644 --- a/main.go +++ b/main.go @@ -5,12 +5,13 @@ import ( "encoding/json" "flag" "fmt" - "github.com/MontFerret/ferret/cli" - "github.com/MontFerret/ferret/cli/browser" - "github.com/MontFerret/ferret/pkg/runtime/core" "io/ioutil" "os" "strings" + + "github.com/MontFerret/ferret/cli" + "github.com/MontFerret/ferret/cli/browser" + "github.com/MontFerret/ferret/pkg/runtime/core" ) type Params []string @@ -55,7 +56,7 @@ var ( conn = flag.String( "cdp", - "http://0.0.0.0:9222", + "", "set CDP address", ) @@ -65,6 +66,12 @@ var ( "launch Chrome", ) + cdpKeepCookies = flag.Bool( + "cdp-keep-cookies", + false, + "keep cookies between queries (i.e. do not open tabs in incognito mode)", + ) + proxyAddress = flag.String( "proxy", "", @@ -153,11 +160,12 @@ func main() { } opts := cli.Options{ - Cdp: cdpConn, - Params: p, - Proxy: *proxyAddress, - UserAgent: *userAgent, - ShowTime: *showTime, + Cdp: cdpConn, + Params: p, + Proxy: *proxyAddress, + UserAgent: *userAgent, + ShowTime: *showTime, + KeepCookies: *cdpKeepCookies, } stat, _ := os.Stdin.Stat() diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index df445299..5507eca9 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -1,12 +1,13 @@ package compiler import ( + "strings" + "github.com/MontFerret/ferret/pkg/parser" "github.com/MontFerret/ferret/pkg/runtime" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/stdlib" "github.com/pkg/errors" - "strings" ) type FqlCompiler struct { @@ -27,9 +28,7 @@ func New(setters ...Option) *FqlCompiler { c.funcs = make(map[string]core.Function) } - return &FqlCompiler{ - stdlib.NewLib(), - } + return c } func (c *FqlCompiler) RegisterFunction(name string, fun core.Function) error { @@ -44,6 +43,10 @@ func (c *FqlCompiler) RegisterFunction(name string, fun core.Function) error { return nil } +func (c *FqlCompiler) RemoveFunction(name string) { + delete(c.funcs, strings.ToUpper(name)) +} + func (c *FqlCompiler) RegisterFunctions(funcs map[string]core.Function) error { for name, fun := range funcs { if err := c.RegisterFunction(name, fun); err != nil { @@ -59,9 +62,6 @@ func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error return nil, ErrEmptyQuery } - p := parser.New(query) - p.AddErrorListener(&errorListener{}) - defer func() { if r := recover(); r != nil { // find out exactly what the error was and set err @@ -78,6 +78,9 @@ func (c *FqlCompiler) Compile(query string) (program *runtime.Program, err error } }() + p := parser.New(query) + p.AddErrorListener(&errorListener{}) + l := newVisitor(query, c.funcs) res := p.Visit(l).(*result) @@ -100,3 +103,10 @@ func (c *FqlCompiler) MustCompile(query string) *runtime.Program { return program } + +func (c *FqlCompiler) RegisteredFunctions() (funcs []string) { + for k := range c.funcs { + funcs = append(funcs, k) + } + return +} diff --git a/pkg/compiler/compiler_collect_test.go b/pkg/compiler/compiler_collect_test.go index c78100c9..9fa2dd1b 100644 --- a/pkg/compiler/compiler_collect_test.go +++ b/pkg/compiler/compiler_collect_test.go @@ -56,6 +56,51 @@ func TestCollect(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Should not have access to variables defined before COLLECT", t, func() { + c := compiler.New() + + _, err := c.Compile(` + LET users = [ + { + active: true, + married: true, + age: 31, + gender: "m" + }, + { + active: true, + married: false, + age: 25, + gender: "f" + }, + { + active: true, + married: false, + age: 36, + gender: "m" + }, + { + active: false, + married: true, + age: 69, + gender: "m" + }, + { + active: true, + married: true, + age: 45, + gender: "f" + } + ] + FOR i IN users + LET x = "foo" + COLLECT gender = i.gender + RETURN {x, gender} + `) + + So(err, ShouldNotBeNil) + }) + Convey("Should group result by a single key", t, func() { c := compiler.New() diff --git a/pkg/compiler/compiler_filter_test.go b/pkg/compiler/compiler_filter_test.go index 17c97c69..051314ad 100644 --- a/pkg/compiler/compiler_filter_test.go +++ b/pkg/compiler/compiler_filter_test.go @@ -3,6 +3,8 @@ package compiler_test import ( "context" "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/values" . "github.com/smartystreets/goconvey/convey" "testing" ) @@ -311,6 +313,60 @@ func TestForFilter(t *testing.T) { So(string(out), ShouldEqual, `[]`) }) + + Convey("Should define variables", t, func() { + c := compiler.New() + + p, err := c.Compile(` + FOR i IN [ 1, 2, 3, 4, 1, 3 ] + LET x = 2 + FILTER i > x + RETURN i + x + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[5,6,5]`) + }) + + Convey("Should call funcions", t, func() { + c := compiler.New() + counterA := 0 + counterB := 0 + c.RegisterFunction("COUNT_A", func(ctx context.Context, args ...core.Value) (core.Value, error) { + counterA++ + + return values.None, nil + }) + + c.RegisterFunction("COUNT_B", func(ctx context.Context, args ...core.Value) (core.Value, error) { + counterB++ + + return values.None, nil + }) + + p, err := c.Compile(` + FOR i IN [ 1, 2, 3, 4, 1, 3 ] + LET x = 2 + COUNT_A() + FILTER i > x + COUNT_B() + RETURN i + x + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + So(counterA, ShouldEqual, 6) + So(counterB, ShouldEqual, 3) + So(string(out), ShouldEqual, `[5,6,5]`) + }) } func BenchmarkFilter(b *testing.B) { diff --git a/pkg/compiler/compiler_for_test.go b/pkg/compiler/compiler_for_test.go index 245829f0..21a0a354 100644 --- a/pkg/compiler/compiler_for_test.go +++ b/pkg/compiler/compiler_for_test.go @@ -248,154 +248,6 @@ func TestFor(t *testing.T) { So(string(out), ShouldEqual, `[1,2,3,4]`) }) - - Convey("Should compile query with LIMIT 2", t, func() { - c := compiler.New() - - p, err := c.Compile(` - FOR i IN [ 1, 2, 3, 4, 1, 3 ] - LIMIT 2 - RETURN i - `) - - So(err, ShouldBeNil) - - out, err := p.Run(context.Background()) - - So(err, ShouldBeNil) - - So(string(out), ShouldEqual, `[1,2]`) - }) - - Convey("Should compile query with LIMIT 2, 2", t, func() { - c := compiler.New() - - // 4 is offset - // 2 is count - p, err := c.Compile(` - FOR i IN [ 1,2,3,4,5,6,7,8 ] - LIMIT 4, 2 - RETURN i - `) - - So(err, ShouldBeNil) - - out, err := p.Run(context.Background()) - - So(err, ShouldBeNil) - - So(string(out), ShouldEqual, `[5,6]`) - }) - - Convey("Should compile query with SORT statement", t, func() { - c := compiler.New() - - p, err := c.Compile(` - LET users = [ - { - active: true, - age: 31, - gender: "m" - }, - { - active: true, - age: 29, - gender: "f" - }, - { - active: true, - age: 36, - gender: "m" - } - ] - FOR u IN users - SORT u.age - RETURN u - `) - - So(err, ShouldBeNil) - - out, err := p.Run(context.Background()) - - So(err, ShouldBeNil) - - So(string(out), ShouldEqual, `[{"active":true,"age":29,"gender":"f"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":36,"gender":"m"}]`) - }) - - Convey("Should compile query with SORT DESC statement", t, func() { - c := compiler.New() - - p, err := c.Compile(` - LET users = [ - { - active: true, - age: 31, - gender: "m" - }, - { - active: true, - age: 29, - gender: "f" - }, - { - active: true, - age: 36, - gender: "m" - } - ] - FOR u IN users - SORT u.age DESC - RETURN u - `) - - So(err, ShouldBeNil) - - out, err := p.Run(context.Background()) - - So(err, ShouldBeNil) - - So(string(out), ShouldEqual, `[{"active":true,"age":36,"gender":"m"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":29,"gender":"f"}]`) - }) - - Convey("Should compile query with SORT statement with multiple expressions", t, func() { - c := compiler.New() - - p, err := c.Compile(` - LET users = [ - { - active: true, - age: 31, - gender: "m" - }, - { - active: true, - age: 29, - gender: "f" - }, - { - active: true, - age: 31, - gender: "f" - }, - { - active: true, - age: 36, - gender: "m" - } - ] - FOR u IN users - SORT u.age, u.gender - RETURN u - `) - - So(err, ShouldBeNil) - - out, err := p.Run(context.Background()) - - So(err, ShouldBeNil) - - So(string(out), ShouldEqual, `[{"active":true,"age":29,"gender":"f"},{"active":true,"age":31,"gender":"f"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":36,"gender":"m"}]`) - }) } func BenchmarkForEmpty(b *testing.B) { diff --git a/pkg/compiler/compiler_let_test.go b/pkg/compiler/compiler_let_test.go index 21111b46..fc0e0236 100644 --- a/pkg/compiler/compiler_let_test.go +++ b/pkg/compiler/compiler_let_test.go @@ -197,4 +197,15 @@ func TestLet(t *testing.T) { So(err, ShouldBeNil) So(string(out), ShouldEqual, "[1,2,3]") }) + + Convey("Should not compile FOR foo IN foo", t, func() { + c := compiler.New() + + _, err := c.Compile(` + FOR foo IN foo + RETURN foo + `) + + So(err, ShouldNotBeNil) + }) } diff --git a/pkg/compiler/compiler_limit_test.go b/pkg/compiler/compiler_limit_test.go new file mode 100644 index 00000000..5c8859e9 --- /dev/null +++ b/pkg/compiler/compiler_limit_test.go @@ -0,0 +1,96 @@ +package compiler_test + +import ( + "context" + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/values" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestForLimit(t *testing.T) { + Convey("Should compile query with LIMIT 2", t, func() { + c := compiler.New() + + p, err := c.Compile(` + FOR i IN [ 1, 2, 3, 4, 1, 3 ] + LIMIT 2 + RETURN i + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[1,2]`) + }) + + Convey("Should compile query with LIMIT 2, 2", t, func() { + c := compiler.New() + + // 4 is offset + // 2 is count + p, err := c.Compile(` + FOR i IN [ 1,2,3,4,5,6,7,8 ] + LIMIT 4, 2 + RETURN i + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[5,6]`) + }) + + Convey("Should define variables and call functions", t, func() { + c := compiler.New() + counter := 0 + c.RegisterFunction("TEST", func(ctx context.Context, args ...core.Value) (core.Value, error) { + counter++ + + So(args[0], ShouldEqual, "foo") + + return values.None, nil + }) + + p, err := c.Compile(` + FOR i IN [ 1,2,3,4,5,6,7,8 ] + LET x = "foo" + TEST(x) + LIMIT 2 + RETURN i + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + So(counter, ShouldEqual, 2) + So(string(out), ShouldEqual, `[1,2]`) + }) + + Convey("Should be able to reuse values from a source", t, func() { + c := compiler.New() + + p, err := c.Compile(` + FOR i IN [ 1,2,3,4,5,6,7,8 ] + LET x = i + LIMIT 2 + RETURN i*x + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + So(string(out), ShouldEqual, `[1,4]`) + }) +} diff --git a/pkg/compiler/compiler_precedence_test.go b/pkg/compiler/compiler_precedence_test.go new file mode 100644 index 00000000..5306be3f --- /dev/null +++ b/pkg/compiler/compiler_precedence_test.go @@ -0,0 +1,65 @@ +package compiler_test + +import ( + "context" + "github.com/MontFerret/ferret/pkg/compiler" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestPrecedence(t *testing.T) { + Convey("Math operators", t, func() { + Convey("2 + 2 * 2", func() { + c := compiler.New() + + p := c.MustCompile(`RETURN 2 + 2 * 2`) + + out := p.MustRun(context.Background()) + + So(string(out), ShouldEqual, "6") + }) + + Convey("2 * 2 + 2", func() { + c := compiler.New() + + p := c.MustCompile(`RETURN 2 * 2 + 2`) + + out := p.MustRun(context.Background()) + + So(string(out), ShouldEqual, "6") + }) + + Convey("2 * (2 + 2)", func() { + c := compiler.New() + + p := c.MustCompile(`RETURN 2 * (2 + 2)`) + + out := p.MustRun(context.Background()) + + So(string(out), ShouldEqual, "8") + }) + }) + + Convey("Logical", t, func() { + Convey("TRUE OR TRUE AND FALSE", func() { + c := compiler.New() + + p := c.MustCompile(`RETURN TRUE OR TRUE AND FALSE`) + + out := p.MustRun(context.Background()) + + So(string(out), ShouldEqual, "true") + }) + + Convey("FALSE AND TRUE OR TRUE", func() { + c := compiler.New() + + p := c.MustCompile(`RETURN FALSE AND TRUE OR TRUE`) + + out := p.MustRun(context.Background()) + + So(string(out), ShouldEqual, "true") + }) + }) +} diff --git a/pkg/compiler/compiler_sort_test.go b/pkg/compiler/compiler_sort_test.go new file mode 100644 index 00000000..eeb45caa --- /dev/null +++ b/pkg/compiler/compiler_sort_test.go @@ -0,0 +1,212 @@ +package compiler_test + +import ( + "context" + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/values" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestForSort(t *testing.T) { + Convey("Should compile query with SORT statement", t, func() { + c := compiler.New() + + p, err := c.Compile(` + LET users = [ + { + active: true, + age: 31, + gender: "m" + }, + { + active: true, + age: 29, + gender: "f" + }, + { + active: true, + age: 36, + gender: "m" + } + ] + FOR u IN users + SORT u.age + RETURN u + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[{"active":true,"age":29,"gender":"f"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":36,"gender":"m"}]`) + }) + + Convey("Should compile query with SORT DESC statement", t, func() { + c := compiler.New() + + p, err := c.Compile(` + LET users = [ + { + active: true, + age: 31, + gender: "m" + }, + { + active: true, + age: 29, + gender: "f" + }, + { + active: true, + age: 36, + gender: "m" + } + ] + FOR u IN users + SORT u.age DESC + RETURN u + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[{"active":true,"age":36,"gender":"m"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":29,"gender":"f"}]`) + }) + + Convey("Should compile query with SORT statement with multiple expressions", t, func() { + c := compiler.New() + + p, err := c.Compile(` + LET users = [ + { + active: true, + age: 31, + gender: "m" + }, + { + active: true, + age: 29, + gender: "f" + }, + { + active: true, + age: 31, + gender: "f" + }, + { + active: true, + age: 36, + gender: "m" + } + ] + FOR u IN users + SORT u.age, u.gender + RETURN u + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, `[{"active":true,"age":29,"gender":"f"},{"active":true,"age":31,"gender":"f"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":36,"gender":"m"}]`) + }) + + Convey("Should define variables and call functions", t, func() { + c := compiler.New() + counter := 0 + c.RegisterFunction("TEST", func(ctx context.Context, args ...core.Value) (core.Value, error) { + counter++ + + So(args[0], ShouldEqual, "foo") + + return values.None, nil + }) + + p, err := c.Compile(` + LET users = [ + { + active: true, + age: 31, + gender: "m" + }, + { + active: true, + age: 29, + gender: "f" + }, + { + active: true, + age: 31, + gender: "f" + }, + { + active: true, + age: 36, + gender: "m" + } + ] + FOR u IN users + LET x = "foo" + TEST(x) + SORT u.age, u.gender + RETURN u + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + So(counter, ShouldEqual, 4) + So(string(out), ShouldEqual, `[{"active":true,"age":29,"gender":"f"},{"active":true,"age":31,"gender":"f"},{"active":true,"age":31,"gender":"m"},{"active":true,"age":36,"gender":"m"}]`) + }) + + Convey("Should be able to reuse values from a source", t, func() { + c := compiler.New() + + p, err := c.Compile(` + LET users = [ + { + active: true, + age: 31, + gender: "m" + }, + { + active: true, + age: 29, + gender: "f" + }, + { + active: true, + age: 31, + gender: "f" + }, + { + active: true, + age: 36, + gender: "m" + } + ] + FOR u IN users + LET x = u.gender + SORT u.age, u.gender + RETURN {u,x} + `) + + So(err, ShouldBeNil) + + out, err := p.Run(context.Background()) + + So(err, ShouldBeNil) + So(string(out), ShouldEqual, `[{"u":{"active":true,"age":29,"gender":"f"},"x":"f"},{"u":{"active":true,"age":31,"gender":"f"},"x":"f"},{"u":{"active":true,"age":31,"gender":"m"},"x":"m"},{"u":{"active":true,"age":36,"gender":"m"},"x":"m"}]`) + }) +} diff --git a/pkg/compiler/compiler_str_test.go b/pkg/compiler/compiler_str_test.go index 849a8617..daa1871c 100644 --- a/pkg/compiler/compiler_str_test.go +++ b/pkg/compiler/compiler_str_test.go @@ -2,6 +2,8 @@ package compiler_test import ( "context" + "encoding/json" + "fmt" "github.com/MontFerret/ferret/pkg/compiler" . "github.com/smartystreets/goconvey/convey" "testing" @@ -20,6 +22,38 @@ BAR So(string(out), ShouldEqual, `"\nFOO\nBAR\n"`) }) + + Convey("Should be possible to use multi line string with nested strings", t, func() { + out := compiler.New(). + MustCompile(fmt.Sprintf(` +RETURN %s + + + + Title + + + Hello world + + %s +`, "`", "`")). + MustRun(context.Background()) + + out, err := json.Marshal(` + + + + Title + + + Hello world + + `) + + So(err, ShouldBeNil) + + So(string(out), ShouldEqual, string(out)) + }) } func BenchmarkStringLiteral(b *testing.B) { diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go new file mode 100644 index 00000000..89c29d96 --- /dev/null +++ b/pkg/compiler/compiler_test.go @@ -0,0 +1,11 @@ +package compiler_test + +import ( + "context" + "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/values" +) + +func NOOP(_ context.Context, args ...core.Value) (core.Value, error) { + return values.None, nil +} diff --git a/pkg/compiler/result.go b/pkg/compiler/result.go index bdeb1990..16928e16 100644 --- a/pkg/compiler/result.go +++ b/pkg/compiler/result.go @@ -7,10 +7,6 @@ type result struct { err error } -func newResult(data interface{}, err error) *result { - return &result{data, err} -} - func newResultFrom(fn visitorFn) *result { out, err := fn() diff --git a/pkg/compiler/scope.go b/pkg/compiler/scope.go index 578f9a97..9acac22e 100644 --- a/pkg/compiler/scope.go +++ b/pkg/compiler/scope.go @@ -2,7 +2,7 @@ package compiler import ( "github.com/MontFerret/ferret/pkg/runtime/core" - "github.com/pkg/errors" + "github.com/MontFerret/ferret/pkg/runtime/values/types" ) type ( @@ -36,24 +36,24 @@ func (s *scope) GetVariable(name string) (core.Type, error) { parents, err := s.parent.GetVariable(name) if err != nil { - return core.NoneType, err + return types.None, err } return parents, nil } - return core.NoneType, core.Error(ErrVariableNotFound, name) + return types.None, core.Error(ErrVariableNotFound, name) } func (s *scope) SetVariable(name string) error { _, exists := s.vars[name] if exists { - return errors.Wrap(ErrVariableNotUnique, name) + return core.Error(ErrVariableNotUnique, name) } // TODO: add type detection - s.vars[name] = core.NoneType + s.vars[name] = types.None return nil } @@ -62,7 +62,7 @@ func (s *scope) RemoveVariable(name string) error { _, exists := s.vars[name] if !exists { - return errors.Wrap(ErrVariableNotFound, name) + return core.Error(ErrVariableNotFound, name) } delete(s.vars, name) @@ -70,6 +70,10 @@ func (s *scope) RemoveVariable(name string) error { return nil } +func (s *scope) ClearVariables() { + s.vars = make(map[string]core.Type) +} + func (s *scope) Fork() *scope { return newScope(s) } diff --git a/pkg/compiler/visitor.go b/pkg/compiler/visitor.go index 11f0441c..abcadc21 100644 --- a/pkg/compiler/visitor.go +++ b/pkg/compiler/visitor.go @@ -2,6 +2,9 @@ package compiler import ( "fmt" + "strconv" + "strings" + "github.com/MontFerret/ferret/pkg/parser/fql" "github.com/MontFerret/ferret/pkg/runtime" "github.com/MontFerret/ferret/pkg/runtime/collections" @@ -12,8 +15,6 @@ import ( "github.com/MontFerret/ferret/pkg/runtime/expressions/operators" "github.com/antlr/antlr4/runtime/Go/antlr" "github.com/pkg/errors" - "strconv" - "strings" ) type ( @@ -49,7 +50,7 @@ func (v *visitor) VisitProgram(ctx *fql.ProgramContext) interface{} { func (v *visitor) doVisitBody(ctx *fql.BodyContext, scope *scope) (core.Expression, error) { statements := ctx.AllBodyStatement() - body := expressions.NewBlockExpression(len(statements) + 1) + body := expressions.NewBodyExpression(len(statements) + 1) for _, stmt := range statements { e, err := v.doVisitBodyStatement(stmt.(*fql.BodyStatementContext), scope) @@ -91,7 +92,7 @@ func (v *visitor) doVisitBodyStatement(ctx *fql.BodyStatementContext, scope *sco return v.doVisitFunctionCallExpression(funcCall.(*fql.FunctionCallExpressionContext), scope) } - return nil, errors.Wrap(ErrInvalidToken, ctx.GetText()) + return nil, core.Error(ErrInvalidToken, ctx.GetText()) } func (v *visitor) doVisitBodyExpression(ctx *fql.BodyExpressionContext, scope *scope) (core.Expression, error) { @@ -160,17 +161,7 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco var keyVarName string parsedClauses := make([]forOption, 0, 10) - valVar := ctx.ForExpressionValueVariable() - valVarName = valVar.GetText() forInScope := scope.Fork() - forInScope.SetVariable(valVarName) - - keyVar := ctx.ForExpressionKeyVariable() - - if keyVar != nil { - keyVarName = keyVar.GetText() - forInScope.SetVariable(keyVarName) - } srcCtx := ctx.ForExpressionSource().(*fql.ForExpressionSourceContext) srcExp, err := v.doVisitForExpressionSource(srcCtx, forInScope) @@ -179,6 +170,23 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco return nil, err } + valVar := ctx.ForExpressionValueVariable() + valVarName = valVar.GetText() + + if err := forInScope.SetVariable(valVarName); err != nil { + return nil, err + } + + keyVar := ctx.ForExpressionKeyVariable() + + if keyVar != nil { + keyVarName = keyVar.GetText() + + if err := forInScope.SetVariable(keyVarName); err != nil { + return nil, err + } + } + src, err := expressions.NewDataSource( v.getSourceMap(srcCtx), valVarName, @@ -186,100 +194,47 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco srcExp, ) - // Clauses. - // We put clauses parsing before parsing the query body because COLLECT clause overrides scope variables - for _, clause := range ctx.AllForExpressionClause() { - clause := clause.(*fql.ForExpressionClauseContext) - - limitCtx := clause.LimitClause() - - if limitCtx != nil { - limit, offset, err := v.createLimit(limitCtx.(*fql.LimitClauseContext)) - - if err != nil { - return nil, err - } - - parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error { - return f.AddLimit(v.getSourceMap(limitCtx), limit, offset) - }) - - continue - } - - filterCtx := clause.FilterClause() - - if filterCtx != nil { - filterExp, err := v.createFilter(filterCtx.(*fql.FilterClauseContext), forInScope) - - if err != nil { - return nil, err - } - - parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error { - return f.AddFilter(v.getSourceMap(filterCtx), filterExp) - }) - - continue - } - - sortCtx := clause.SortClause() - - if sortCtx != nil { - sortCtx := sortCtx.(*fql.SortClauseContext) - sortExps, err := v.createSort(sortCtx, forInScope) - - if err != nil { - return nil, err - } - - parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error { - return f.AddSort(v.getSourceMap(sortCtx), sortExps...) - }) - } - - collectCtx := clause.CollectClause() - - if collectCtx != nil { - collectCtx := collectCtx.(*fql.CollectClauseContext) - - params, err := v.createCollect(collectCtx, forInScope, valVarName) - - if err != nil { - return nil, err - } - - forInScope.RemoveVariable(valVarName) - - if keyVarName != "" { - forInScope.RemoveVariable(keyVarName) - } - - parsedClauses = append(parsedClauses, func(f *expressions.ForExpression) error { - return f.AddCollect(v.getSourceMap(collectCtx), params) - }) - } + if err != nil { + return nil, err } - body := ctx.AllForExpressionBody() - predicate := expressions.NewBlockExpression(len(body) + 1) + // Clauses. + // We put clauses parsing before parsing the query body because COLLECT clause overrides scope variables + for _, e := range ctx.AllForExpressionBody() { + e := e.(*fql.ForExpressionBodyContext) + clauseCtx := e.ForExpressionClause() + statementCtx := e.ForExpressionStatement() - for _, el := range body { - el, err := v.doVisitForExpressionBody(el.(*fql.ForExpressionBodyContext), forInScope) + if clauseCtx != nil { + setter, err := v.doVisitForExpressionClause( + clauseCtx.(*fql.ForExpressionClauseContext), + forInScope, + valVarName, + keyVarName, + ) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - err = predicate.Add(el) + parsedClauses = append(parsedClauses, setter) + } else if statementCtx != nil { + exp, err := v.doVisitForExpressionStatement( + statementCtx.(*fql.ForExpressionStatementContext), + forInScope, + ) - if err != nil { - return nil, err + if err != nil { + return nil, err + } + + parsedClauses = append(parsedClauses, exp) } } var spread bool var distinct bool + var predicate core.Expression forRetCtx := ctx.ForExpressionReturn().(*fql.ForExpressionReturnContext) returnCtx := forRetCtx.ReturnExpression() @@ -297,7 +252,7 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco distinct = true } - predicate.Add(returnExp) + predicate = returnExp } else { forInCtx := forRetCtx.ForExpression().(*fql.ForExpressionContext) forInExp, err := v.doVisitForExpression(forInCtx, forInScope) @@ -308,7 +263,7 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco spread = true - predicate.Add(forInExp) + predicate = forInExp } forExp, err := expressions.NewForExpression( @@ -333,41 +288,57 @@ func (v *visitor) doVisitForExpression(ctx *fql.ForExpressionContext, scope *sco return forExp, nil } -func (v *visitor) createLimit(ctx *fql.LimitClauseContext) (int, int, error) { +func (v *visitor) doVisitLimitClause(ctx *fql.LimitClauseContext, scope *scope) (core.Expression, core.Expression, error) { var err error - var count int - var offset int + var count core.Expression + var offset core.Expression - intLiterals := ctx.AllIntegerLiteral() + clauseValues := ctx.AllLimitClauseValue() - if len(intLiterals) > 1 { - offset, err = v.parseInt(intLiterals[0]) + if len(clauseValues) > 1 { + offset, err = v.doVisitLimitClauseValue(clauseValues[0].(*fql.LimitClauseValueContext), scope) if err != nil { - return 0, 0, err + return nil, nil, err } - count, err = v.parseInt(intLiterals[1]) + count, err = v.doVisitLimitClauseValue(clauseValues[1].(*fql.LimitClauseValueContext), scope) if err != nil { - return 0, 0, err + return nil, nil, err } } else { - count, err = strconv.Atoi(intLiterals[0].GetText()) + count, err = v.doVisitLimitClauseValue(clauseValues[0].(*fql.LimitClauseValueContext), scope) if err != nil { - return 0, 0, err + return nil, nil, err } + + offset = literals.NewIntLiteral(0) } return count, offset, nil } -func (v *visitor) parseInt(node antlr.TerminalNode) (int, error) { - return strconv.Atoi(node.GetText()) +func (v *visitor) doVisitLimitClauseValue(ctx *fql.LimitClauseValueContext, scope *scope) (core.Expression, error) { + literalCtx := ctx.IntegerLiteral() + + if literalCtx != nil { + i, err := strconv.Atoi(literalCtx.GetText()) + + if err != nil { + return nil, err + } + + return literals.NewIntLiteral(i), nil + } + + paramCtx := ctx.Param() + + return v.doVisitParamContext(paramCtx.(*fql.ParamContext), scope) } -func (v *visitor) createFilter(ctx *fql.FilterClauseContext, scope *scope) (core.Expression, error) { +func (v *visitor) doVisitFilterClause(ctx *fql.FilterClauseContext, scope *scope) (core.Expression, error) { exp := ctx.Expression().(*fql.ExpressionContext) exps, err := v.doVisitAllExpressions(exp.AllExpression(), scope) @@ -386,10 +357,16 @@ func (v *visitor) createFilter(ctx *fql.FilterClauseContext, scope *scope) (core return operators.NewEqualityOperator(v.getSourceMap(ctx), left, right, equalityOp.GetText()) } - logicalOp := exp.LogicalOperator() + logicalAndOp := exp.LogicalAndOperator() - if logicalOp != nil { - return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalOp.GetText()) + if logicalAndOp != nil { + return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalAndOp.GetText()) + } + + logicalOrOp := exp.LogicalOrOperator() + + if logicalOrOp != nil { + return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, logicalOrOp.GetText()) } } else { // should be unary operator @@ -399,7 +376,7 @@ func (v *visitor) createFilter(ctx *fql.FilterClauseContext, scope *scope) (core return nil, core.Error(ErrInvalidToken, ctx.GetText()) } -func (v *visitor) createSort(ctx *fql.SortClauseContext, scope *scope) ([]*clauses.SorterExpression, error) { +func (v *visitor) doVisitSortClause(ctx *fql.SortClauseContext, scope *scope) ([]*clauses.SorterExpression, error) { sortExpCtxs := ctx.AllSortClauseExpression() res := make([]*clauses.SorterExpression, len(sortExpCtxs)) @@ -434,12 +411,13 @@ func (v *visitor) createSort(ctx *fql.SortClauseContext, scope *scope) ([]*claus return res, nil } -func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, valVarName string) (*clauses.Collect, error) { +func (v *visitor) doVisitCollectClause(ctx *fql.CollectClauseContext, scope *scope, valVarName string) (*clauses.Collect, error) { var err error var selectors []*clauses.CollectSelector var projection *clauses.CollectProjection var count *clauses.CollectCount var aggregate *clauses.CollectAggregate + variables := make([]string, 0, 10) groupingCtx := ctx.CollectGrouping() @@ -452,17 +430,14 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val selectors = make([]*clauses.CollectSelector, 0, len(collectSelectors)) for _, cs := range collectSelectors { - selector, err := v.createCollectSelector(cs.(*fql.CollectSelectorContext), scope) + selector, err := v.doVisitCollectSelector(cs.(*fql.CollectSelectorContext), scope) if err != nil { return nil, err } selectors = append(selectors, selector) - - if err := scope.SetVariable(selector.Variable()); err != nil { - return nil, err - } + variables = append(variables, selector.Variable()) } } @@ -475,7 +450,7 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val // if projection expression is defined like WITH group = { foo: i.bar } if projectionSelectorCtx != nil { - selector, err := v.createCollectSelector(projectionSelectorCtx.(*fql.CollectSelectorContext), scope) + selector, err := v.doVisitCollectSelector(projectionSelectorCtx.(*fql.CollectSelectorContext), scope) if err != nil { return nil, err @@ -521,10 +496,7 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val } if projectionSelector != nil { - if err := scope.SetVariable(projectionSelector.Variable()); err != nil { - return nil, err - } - + variables = append(variables, projectionSelector.Variable()) projection, err = clauses.NewCollectProjection(projectionSelector) if err != nil { @@ -539,10 +511,7 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val if countCtx != nil { countCtx := countCtx.(*fql.CollectCounterContext) variable := countCtx.Identifier().GetText() - - if err := scope.SetVariable(variable); err != nil { - return nil, err - } + variables = append(variables, variable) count, err = clauses.NewCollectCount(variable) @@ -560,13 +529,14 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val selectors := make([]*clauses.CollectAggregateSelector, 0, len(selectorCtxs)) for _, sc := range selectorCtxs { - selector, err := v.createCollectAggregateSelector(sc.(*fql.CollectAggregateSelectorContext), scope) + selector, err := v.doVisitCollectAggregateSelector(sc.(*fql.CollectAggregateSelectorContext), scope) if err != nil { return nil, err } selectors = append(selectors, selector) + variables = append(variables, selector.Variable()) } aggregate, err = clauses.NewCollectAggregate(selectors) @@ -576,10 +546,19 @@ func (v *visitor) createCollect(ctx *fql.CollectClauseContext, scope *scope, val } } + // clear all variables defined before + scope.ClearVariables() + + for _, variable := range variables { + if err := scope.SetVariable(variable); err != nil { + return nil, err + } + } + return clauses.NewCollect(selectors, projection, count, aggregate) } -func (v *visitor) createCollectSelector(ctx *fql.CollectSelectorContext, scope *scope) (*clauses.CollectSelector, error) { +func (v *visitor) doVisitCollectSelector(ctx *fql.CollectSelectorContext, scope *scope) (*clauses.CollectSelector, error) { variable := ctx.Identifier().GetText() exp, err := v.doVisitExpression(ctx.Expression().(*fql.ExpressionContext), scope) @@ -590,7 +569,7 @@ func (v *visitor) createCollectSelector(ctx *fql.CollectSelectorContext, scope * return clauses.NewCollectSelector(variable, exp) } -func (v *visitor) createCollectAggregateSelector(ctx *fql.CollectAggregateSelectorContext, scope *scope) (*clauses.CollectAggregateSelector, error) { +func (v *visitor) doVisitCollectAggregateSelector(ctx *fql.CollectAggregateSelectorContext, scope *scope) (*clauses.CollectAggregateSelector, error) { variable := ctx.Identifier().GetText() fnCtx := ctx.FunctionCallExpression() @@ -607,10 +586,6 @@ func (v *visitor) createCollectAggregateSelector(ctx *fql.CollectAggregateSelect return nil, core.Error(core.ErrInvalidType, "expected function expression") } - if err := scope.SetVariable(variable); err != nil { - return nil, err - } - return clauses.NewCollectAggregateSelector(variable, fnExp.Arguments(), fnExp.Function()) } @@ -663,17 +638,101 @@ func (v *visitor) doVisitForExpressionSource(ctx *fql.ForExpressionSourceContext return nil, core.Error(ErrInvalidDataSource, ctx.GetText()) } -func (v *visitor) doVisitForExpressionBody(ctx *fql.ForExpressionBodyContext, scope *scope) (core.Expression, error) { - varDecCtx := ctx.VariableDeclaration() +func (v *visitor) doVisitForExpressionClause(ctx *fql.ForExpressionClauseContext, scope *scope, valVarName, _ string) (func(f *expressions.ForExpression) error, error) { + limitCtx := ctx.LimitClause() - if varDecCtx != nil { - return v.doVisitVariableDeclaration(varDecCtx.(*fql.VariableDeclarationContext), scope) + if limitCtx != nil { + limit, offset, err := v.doVisitLimitClause(limitCtx.(*fql.LimitClauseContext), scope) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddLimit(v.getSourceMap(limitCtx), limit, offset) + }, nil } - funcCallCtx := ctx.FunctionCallExpression() + filterCtx := ctx.FilterClause() - if funcCallCtx != nil { - return v.doVisitFunctionCallExpression(funcCallCtx.(*fql.FunctionCallExpressionContext), scope) + if filterCtx != nil { + filterExp, err := v.doVisitFilterClause(filterCtx.(*fql.FilterClauseContext), scope) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddFilter(v.getSourceMap(filterCtx), filterExp) + }, nil + } + + sortCtx := ctx.SortClause() + + if sortCtx != nil { + sortCtx := sortCtx.(*fql.SortClauseContext) + sortExps, err := v.doVisitSortClause(sortCtx, scope) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddSort(v.getSourceMap(sortCtx), sortExps...) + }, nil + } + + collectCtx := ctx.CollectClause() + + if collectCtx != nil { + collectCtx := collectCtx.(*fql.CollectClauseContext) + params, err := v.doVisitCollectClause(collectCtx, scope, valVarName) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddCollect(v.getSourceMap(collectCtx), params) + }, nil + } + + return nil, v.unexpectedToken(ctx) +} + +func (v *visitor) doVisitForExpressionStatement(ctx *fql.ForExpressionStatementContext, scope *scope) (func(f *expressions.ForExpression) error, error) { + variableCtx := ctx.VariableDeclaration() + + if variableCtx != nil { + variableExp, err := v.doVisitVariableDeclaration( + variableCtx.(*fql.VariableDeclarationContext), + scope, + ) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddStatement(variableExp) + }, nil + } + + fnCallCtx := ctx.FunctionCallExpression() + + if fnCallCtx != nil { + fnCallExp, err := v.doVisitFunctionCallExpression( + fnCallCtx.(*fql.FunctionCallExpressionContext), + scope, + ) + + if err != nil { + return nil, err + } + + return func(f *expressions.ForExpression) error { + return f.AddStatement(fnCallExp) + }, nil } return nil, v.unexpectedToken(ctx) @@ -756,11 +815,12 @@ func (v *visitor) doVisitObjectLiteral(ctx *fql.ObjectLiteralContext, scope *sco computedProp := assignment.ComputedPropertyName() shortHand := assignment.ShorthandPropertyName() - if prop != nil { + switch { + case prop != nil: name, err = v.doVisitPropertyNameContext(prop.(*fql.PropertyNameContext), scope) - } else if computedProp != nil { + case computedProp != nil: name, err = v.doVisitComputedPropertyNameContext(computedProp.(*fql.ComputedPropertyNameContext), scope) - } else { + default: name, err = v.doVisitShorthandPropertyNameContext(shortHand.(*fql.ShorthandPropertyNameContext), scope) } @@ -874,14 +934,22 @@ func (v *visitor) doVisitIntegerLiteral(ctx *fql.IntegerLiteralContext) (core.Ex } func (v *visitor) doVisitStringLiteral(ctx *fql.StringLiteralContext) (core.Expression, error) { - text := ctx.StringLiteral().GetText() + var text string + + strLiteral := ctx.StringLiteral() + + if strLiteral != nil { + text = strLiteral.GetText() + } else { + text = ctx.TemplateStringLiteral().GetText() + } // remove extra quotes return literals.NewStringLiteral(text[1 : len(text)-1]), nil } func (v *visitor) doVisitBooleanLiteral(ctx *fql.BooleanLiteralContext) (core.Expression, error) { - return literals.NewBooleanLiteral(strings.ToUpper(ctx.GetText()) == "TRUE"), nil + return literals.NewBooleanLiteral(strings.EqualFold(ctx.GetText(), "TRUE")), nil } func (v *visitor) doVisitNoneLiteral(_ *fql.NoneLiteralContext) (core.Expression, error) { @@ -950,7 +1018,7 @@ func (v *visitor) doVisitRangeOperator(ctx *fql.RangeOperatorContext, scope *sco } if len(exp) < 2 { - return nil, errors.Wrap(ErrInvalidToken, ctx.GetText()) + return nil, core.Error(ErrInvalidToken, ctx.GetText()) } left := exp[0] @@ -1022,7 +1090,21 @@ func (v *visitor) doVisitAllExpressions(contexts []fql.IExpressionContext, scope } func (v *visitor) doVisitMathOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) { - mathOp := ctx.MathOperator().(*fql.MathOperatorContext) + var operator operators.MathOperatorType + multiCtx := ctx.MultiplicativeOperator() + + if multiCtx != nil { + operator = operators.MathOperatorType(multiCtx.GetText()) + } else { + additiveCtx := ctx.AdditiveOperator() + + if additiveCtx == nil { + return nil, ErrInvalidToken + } + + operator = operators.MathOperatorType(additiveCtx.GetText()) + } + exps, err := v.doVisitAllExpressions(ctx.AllExpression(), scope) if err != nil { @@ -1033,10 +1115,10 @@ func (v *visitor) doVisitMathOperator(ctx *fql.ExpressionContext, scope *scope) right := exps[1] return operators.NewMathOperator( - v.getSourceMap(mathOp), + v.getSourceMap(ctx), left, right, - operators.MathOperatorType(mathOp.GetText()), + operator, ) } @@ -1059,7 +1141,22 @@ func (v *visitor) doVisitUnaryOperator(ctx *fql.ExpressionContext, scope *scope) } func (v *visitor) doVisitLogicalOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) { - logicalOp := ctx.LogicalOperator().(*fql.LogicalOperatorContext) + var operator string + + logicalAndOp := ctx.LogicalAndOperator() + + if logicalAndOp != nil { + operator = logicalAndOp.GetText() + } else { + logicalOrOp := ctx.LogicalOrOperator() + + if logicalOrOp == nil { + return nil, ErrInvalidToken + } + + operator = logicalOrOp.GetText() + } + exps, err := v.doVisitAllExpressions(ctx.AllExpression(), scope) if err != nil { @@ -1069,7 +1166,7 @@ func (v *visitor) doVisitLogicalOperator(ctx *fql.ExpressionContext, scope *scop left := exps[0] right := exps[1] - return operators.NewLogicalOperator(v.getSourceMap(logicalOp), left, right, logicalOp.GetText()) + return operators.NewLogicalOperator(v.getSourceMap(ctx), left, right, operator) } func (v *visitor) doVisitEqualityOperator(ctx *fql.ExpressionContext, scope *scope) (core.OperatorExpression, error) { @@ -1114,14 +1211,19 @@ func (v *visitor) doVisitArrayOperator(ctx *fql.ExpressionContext, scope *scope) var comparator core.OperatorExpression var err error - if ctx.InOperator() != nil { + switch { + case ctx.InOperator() != nil: comparator, err = v.doVisitInOperator(ctx, scope) - } else if ctx.EqualityOperator() != nil { + case ctx.EqualityOperator() != nil: comparator, err = v.doVisitEqualityOperator(ctx, scope) - } else { + default: return nil, v.unexpectedToken(ctx) } + if err != nil { + return nil, err + } + exps, err := v.doVisitAllExpressions(ctx.AllExpression(), scope) if err != nil { @@ -1150,13 +1252,83 @@ func (v *visitor) doVisitArrayOperator(ctx *fql.ExpressionContext, scope *scope) ) } +func (v *visitor) doVisitExpressionGroup(ctx *fql.ExpressionGroupContext, scope *scope) (core.Expression, error) { + exp := ctx.Expression() + + if exp == nil { + return nil, ErrInvalidToken + } + + return v.doVisitExpression(exp.(*fql.ExpressionContext), scope) +} + func (v *visitor) doVisitExpression(ctx *fql.ExpressionContext, scope *scope) (core.Expression, error) { + seq := ctx.ExpressionGroup() + + if seq != nil { + return v.doVisitExpressionGroup(seq.(*fql.ExpressionGroupContext), scope) + } + + member := ctx.MemberExpression() + + if member != nil { + return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), scope) + } + + funCall := ctx.FunctionCallExpression() + + if funCall != nil { + return v.doVisitFunctionCallExpression(funCall.(*fql.FunctionCallExpressionContext), scope) + } + notOp := ctx.UnaryOperator() if notOp != nil { return v.doVisitUnaryOperator(ctx, scope) } + multiOp := ctx.MultiplicativeOperator() + + if multiOp != nil { + return v.doVisitMathOperator(ctx, scope) + } + + addOp := ctx.AdditiveOperator() + + if addOp != nil { + return v.doVisitMathOperator(ctx, scope) + } + + arrOp := ctx.ArrayOperator() + + if arrOp != nil { + return v.doVisitArrayOperator(ctx, scope) + } + + equalityOp := ctx.EqualityOperator() + + if equalityOp != nil { + return v.doVisitEqualityOperator(ctx, scope) + } + + inOp := ctx.InOperator() + + if inOp != nil { + return v.doVisitInOperator(ctx, scope) + } + + logicalAndOp := ctx.LogicalAndOperator() + + if logicalAndOp != nil { + return v.doVisitLogicalOperator(ctx, scope) + } + + logicalOrOp := ctx.LogicalOrOperator() + + if logicalOrOp != nil { + return v.doVisitLogicalOperator(ctx, scope) + } + variable := ctx.Variable() if variable != nil { @@ -1199,54 +1371,12 @@ func (v *visitor) doVisitExpression(ctx *fql.ExpressionContext, scope *scope) (c return v.doVisitObjectLiteral(obj.(*fql.ObjectLiteralContext), scope) } - funCall := ctx.FunctionCallExpression() - - if funCall != nil { - return v.doVisitFunctionCallExpression(funCall.(*fql.FunctionCallExpressionContext), scope) - } - - member := ctx.MemberExpression() - - if member != nil { - return v.doVisitMemberExpression(member.(*fql.MemberExpressionContext), scope) - } - none := ctx.NoneLiteral() if none != nil { return v.doVisitNoneLiteral(none.(*fql.NoneLiteralContext)) } - arrOp := ctx.ArrayOperator() - - if arrOp != nil { - return v.doVisitArrayOperator(ctx, scope) - } - - inOp := ctx.InOperator() - - if inOp != nil { - return v.doVisitInOperator(ctx, scope) - } - - equalityOp := ctx.EqualityOperator() - - if equalityOp != nil { - return v.doVisitEqualityOperator(ctx, scope) - } - - logicalOp := ctx.LogicalOperator() - - if logicalOp != nil { - return v.doVisitLogicalOperator(ctx, scope) - } - - mathOp := ctx.MathOperator() - - if mathOp != nil { - return v.doVisitMathOperator(ctx, scope) - } - questionCtx := ctx.QuestionMark() if questionCtx != nil { @@ -1311,37 +1441,37 @@ func (v *visitor) visit(node antlr.Tree, scope *scope) (core.Expression, error) var out core.Expression var err error - switch node.(type) { + switch ctx := node.(type) { case *fql.BodyContext: - out, err = v.doVisitBody(node.(*fql.BodyContext), scope) + out, err = v.doVisitBody(ctx, scope) case *fql.ExpressionContext: - out, err = v.doVisitExpression(node.(*fql.ExpressionContext), scope) + out, err = v.doVisitExpression(ctx, scope) case *fql.ForExpressionContext: - out, err = v.doVisitForExpression(node.(*fql.ForExpressionContext), scope) + out, err = v.doVisitForExpression(ctx, scope) case *fql.ReturnExpressionContext: - out, err = v.doVisitReturnExpression(node.(*fql.ReturnExpressionContext), scope) + out, err = v.doVisitReturnExpression(ctx, scope) case *fql.ArrayLiteralContext: - out, err = v.doVisitArrayLiteral(node.(*fql.ArrayLiteralContext), scope) + out, err = v.doVisitArrayLiteral(ctx, scope) case *fql.ObjectLiteralContext: - out, err = v.doVisitObjectLiteral(node.(*fql.ObjectLiteralContext), scope) + out, err = v.doVisitObjectLiteral(ctx, scope) case *fql.StringLiteralContext: - out, err = v.doVisitStringLiteral(node.(*fql.StringLiteralContext)) + out, err = v.doVisitStringLiteral(ctx) case *fql.IntegerLiteralContext: - out, err = v.doVisitIntegerLiteral(node.(*fql.IntegerLiteralContext)) + out, err = v.doVisitIntegerLiteral(ctx) case *fql.FloatLiteralContext: - out, err = v.doVisitFloatLiteral(node.(*fql.FloatLiteralContext)) + out, err = v.doVisitFloatLiteral(ctx) case *fql.BooleanLiteralContext: - out, err = v.doVisitBooleanLiteral(node.(*fql.BooleanLiteralContext)) + out, err = v.doVisitBooleanLiteral(ctx) case *fql.NoneLiteralContext: - out, err = v.doVisitNoneLiteral(node.(*fql.NoneLiteralContext)) + out, err = v.doVisitNoneLiteral(ctx) case *fql.VariableContext: - out, err = v.doVisitVariable(node.(*fql.VariableContext), scope) + out, err = v.doVisitVariable(ctx, scope) case *fql.VariableDeclarationContext: - out, err = v.doVisitVariableDeclaration(node.(*fql.VariableDeclarationContext), scope) + out, err = v.doVisitVariableDeclaration(ctx, scope) case *fql.FunctionCallExpressionContext: - out, err = v.doVisitFunctionCallExpression(node.(*fql.FunctionCallExpressionContext), scope) + out, err = v.doVisitFunctionCallExpression(ctx, scope) case *fql.ParamContext: - out, err = v.doVisitParamContext(node.(*fql.ParamContext), scope) + out, err = v.doVisitParamContext(ctx, scope) default: err = v.unexpectedToken(node) } diff --git a/pkg/drivers/cdp/document.go b/pkg/drivers/cdp/document.go new file mode 100644 index 00000000..682d2469 --- /dev/null +++ b/pkg/drivers/cdp/document.go @@ -0,0 +1,1126 @@ +package cdp + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "sync" + "time" + + "github.com/MontFerret/ferret/pkg/drivers" + "github.com/MontFerret/ferret/pkg/drivers/cdp/eval" + "github.com/MontFerret/ferret/pkg/drivers/cdp/events" + "github.com/MontFerret/ferret/pkg/drivers/cdp/templates" + "github.com/MontFerret/ferret/pkg/drivers/common" + "github.com/MontFerret/ferret/pkg/runtime/core" + "github.com/MontFerret/ferret/pkg/runtime/logging" + "github.com/MontFerret/ferret/pkg/runtime/values" + "github.com/MontFerret/ferret/pkg/runtime/values/types" + "github.com/mafredri/cdp" + "github.com/mafredri/cdp/protocol/dom" + "github.com/mafredri/cdp/protocol/input" + "github.com/mafredri/cdp/protocol/network" + "github.com/mafredri/cdp/protocol/page" + "github.com/mafredri/cdp/rpcc" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +const BlankPageURL = "about:blank" + +type HTMLDocument struct { + sync.Mutex + logger *zerolog.Logger + conn *rpcc.Conn + client *cdp.Client + events *events.EventBroker + url values.String + element *HTMLElement +} + +func handleLoadError(logger *zerolog.Logger, client *cdp.Client) { + err := client.Page.Close(context.Background()) + + if err != nil { + logger.Warn().Timestamp().Err(err).Msg("unabled to close document on load error") + } +} + +func LoadHTMLDocument( + ctx context.Context, + conn *rpcc.Conn, + client *cdp.Client, + params drivers.LoadDocumentParams, +) (drivers.HTMLDocument, error) { + logger := logging.FromContext(ctx) + + if conn == nil { + return nil, core.Error(core.ErrMissedArgument, "connection") + } + + if params.URL == "" { + return nil, core.Error(core.ErrMissedArgument, "url") + } + + if params.Cookies != nil { + cookies := make([]network.CookieParam, 0, len(params.Cookies)) + + for _, c := range params.Cookies { + cookies = append(cookies, fromDriverCookie(params.URL, c)) + + logger. + Debug(). + Timestamp(). + Str("cookie", c.Name). + Msg("set cookie") + } + + err := client.Network.SetCookies( + ctx, + network.NewSetCookiesArgs(cookies), + ) + + if err != nil { + return nil, err + } + } + + if params.Header != nil { + j, err := json.Marshal(params.Header) + + if err != nil { + return nil, err + } + + for k := range params.Header { + logger. + Debug(). + Timestamp(). + Str("header", k). + Msg("set header") + } + + err = client.Network.SetExtraHTTPHeaders( + ctx, + network.NewSetExtraHTTPHeadersArgs(network.Headers(j)), + ) + + if err != nil { + return nil, err + } + } + + var err error + + if params.URL != BlankPageURL { + err = waitForLoadEvent(ctx, client) + + if err != nil { + handleLoadError(logger, client) + + return nil, err + } + } + + node, err := getRootElement(ctx, client) + + if err != nil { + handleLoadError(logger, client) + return nil, errors.Wrap(err, "failed to get root element") + } + + broker, err := createEventBroker(client) + + if err != nil { + handleLoadError(logger, client) + return nil, errors.Wrap(err, "failed to create event events") + } + + rootElement, err := LoadElement( + ctx, + logger, + client, + broker, + node.Root.NodeID, + node.Root.BackendNodeID, + ) + + if err != nil { + broker.Stop() + broker.Close() + handleLoadError(logger, client) + + return nil, errors.Wrap(err, "failed to load root element") + } + + return NewHTMLDocument( + logger, + conn, + client, + broker, + values.NewString(params.URL), + rootElement, + ), nil +} + +func NewHTMLDocument( + logger *zerolog.Logger, + conn *rpcc.Conn, + client *cdp.Client, + broker *events.EventBroker, + url values.String, + rootElement *HTMLElement, +) *HTMLDocument { + doc := new(HTMLDocument) + doc.logger = logger + doc.conn = conn + doc.client = client + doc.events = broker + doc.url = url + doc.element = rootElement + + broker.AddEventListener(events.EventLoad, doc.handlePageLoad) + broker.AddEventListener(events.EventError, doc.handleError) + + return doc +} + +func (doc *HTMLDocument) MarshalJSON() ([]byte, error) { + doc.Lock() + defer doc.Unlock() + + return doc.element.MarshalJSON() +} + +func (doc *HTMLDocument) Type() core.Type { + return drivers.HTMLDocumentType +} + +func (doc *HTMLDocument) String() string { + doc.Lock() + defer doc.Unlock() + + return doc.url.String() +} + +func (doc *HTMLDocument) Unwrap() interface{} { + doc.Lock() + defer doc.Unlock() + + return doc.element +} + +func (doc *HTMLDocument) Hash() uint64 { + doc.Lock() + defer doc.Unlock() + + h := fnv.New64a() + + h.Write([]byte(doc.Type().String())) + h.Write([]byte(":")) + h.Write([]byte(doc.url)) + + return h.Sum64() +} + +func (doc *HTMLDocument) Copy() core.Value { + return values.None +} + +func (doc *HTMLDocument) Compare(other core.Value) int64 { + doc.Lock() + defer doc.Unlock() + + switch other.Type() { + case drivers.HTMLDocumentType: + other := other.(drivers.HTMLDocument) + + return doc.url.Compare(other.GetURL()) + default: + return drivers.Compare(doc.Type(), other.Type()) + } +} + +func (doc *HTMLDocument) Iterate(ctx context.Context) (core.Iterator, error) { + return doc.element.Iterate(ctx) +} + +func (doc *HTMLDocument) GetIn(ctx context.Context, path []core.Value) (core.Value, error) { + return common.GetInDocument(ctx, doc, path) +} + +func (doc *HTMLDocument) SetIn(ctx context.Context, path []core.Value, value core.Value) error { + return common.SetInDocument(ctx, doc, path, value) +} + +func (doc *HTMLDocument) Close() error { + doc.Lock() + defer doc.Unlock() + + var err error + + err = doc.events.Stop() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to stop event events") + } + + err = doc.events.Close() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close event events") + } + + err = doc.element.Close() + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close root element") + } + + err = doc.client.Page.Close(context.Background()) + + if err != nil { + doc.logger.Warn(). + Timestamp(). + Str("url", doc.url.String()). + Err(err). + Msg("failed to close browser page") + } + + return doc.conn.Close() +} + +func (doc *HTMLDocument) NodeType() values.Int { + doc.Lock() + defer doc.Unlock() + + return doc.element.NodeType() +} + +func (doc *HTMLDocument) NodeName() values.String { + doc.Lock() + defer doc.Unlock() + + return doc.element.NodeName() +} + +func (doc *HTMLDocument) Length() values.Int { + doc.Lock() + defer doc.Unlock() + + return doc.element.Length() +} + +func (doc *HTMLDocument) GetChildNodes(ctx context.Context) core.Value { + doc.Lock() + defer doc.Unlock() + + return doc.element.GetChildNodes(ctx) +} + +func (doc *HTMLDocument) GetChildNode(ctx context.Context, idx values.Int) core.Value { + doc.Lock() + defer doc.Unlock() + + return doc.element.GetChildNode(ctx, idx) +} + +func (doc *HTMLDocument) QuerySelector(ctx context.Context, selector values.String) core.Value { + doc.Lock() + defer doc.Unlock() + + return doc.element.QuerySelector(ctx, selector) +} + +func (doc *HTMLDocument) QuerySelectorAll(ctx context.Context, selector values.String) core.Value { + doc.Lock() + defer doc.Unlock() + + return doc.element.QuerySelectorAll(ctx, selector) +} + +func (doc *HTMLDocument) DocumentElement() drivers.HTMLElement { + doc.Lock() + defer doc.Unlock() + + return doc.element +} + +func (doc *HTMLDocument) GetURL() core.Value { + doc.Lock() + defer doc.Unlock() + + return doc.url +} + +func (doc *HTMLDocument) GetCookies(ctx context.Context) (*values.Array, error) { + doc.Lock() + defer doc.Unlock() + + repl, err := doc.client.Network.GetAllCookies(ctx) + + if err != nil { + return values.NewArray(0), err + } + + if repl.Cookies == nil { + return values.NewArray(0), nil + } + + cookies := values.NewArray(len(repl.Cookies)) + + for _, c := range repl.Cookies { + cookies.Push(toDriverCookie(c)) + } + + return cookies, nil +} + +func (doc *HTMLDocument) SetCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error { + doc.Lock() + defer doc.Unlock() + + if len(cookies) == 0 { + return nil + } + + params := make([]network.CookieParam, 0, len(cookies)) + + for _, c := range cookies { + params = append(params, fromDriverCookie(doc.url.String(), c)) + } + + return doc.client.Network.SetCookies(ctx, network.NewSetCookiesArgs(params)) +} + +func (doc *HTMLDocument) DeleteCookies(ctx context.Context, cookies ...drivers.HTTPCookie) error { + doc.Lock() + defer doc.Unlock() + + if len(cookies) == 0 { + return nil + } + + var err error + + for _, c := range cookies { + err = doc.client.Network.DeleteCookies(ctx, fromDriverCookieDelete(doc.url.String(), c)) + + if err != nil { + break + } + } + + return err +} + +func (doc *HTMLDocument) SetURL(ctx context.Context, url values.String) error { + return doc.Navigate(ctx, url) +} + +func (doc *HTMLDocument) CountBySelector(ctx context.Context, selector values.String) values.Int { + doc.Lock() + defer doc.Unlock() + + return doc.element.CountBySelector(ctx, selector) +} + +func (doc *HTMLDocument) ExistsBySelector(ctx context.Context, selector values.String) values.Boolean { + doc.Lock() + defer doc.Unlock() + + return doc.element.ExistsBySelector(ctx, selector) +} + +func (doc *HTMLDocument) ClickBySelector(ctx context.Context, selector values.String) (values.Boolean, error) { + res, err := eval.Eval( + ctx, + doc.client, + fmt.Sprintf(` + var el = document.querySelector(%s); + if (el == null) { + return false; + } + var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true }); + el.dispatchEvent(evt); + return true; + `, eval.ParamString(selector.String())), + true, + false, + ) + + if err != nil { + return values.False, err + } + + if res.Type() == types.Boolean { + return res.(values.Boolean), nil + } + + return values.False, nil +} + +func (doc *HTMLDocument) ClickBySelectorAll(ctx context.Context, selector values.String) (values.Boolean, error) { + res, err := eval.Eval( + ctx, + doc.client, + fmt.Sprintf(` + var elements = document.querySelectorAll(%s); + if (elements == null) { + return false; + } + elements.forEach((el) => { + var evt = new window.MouseEvent('click', { bubbles: true, cancelable: true }); + el.dispatchEvent(evt); + }); + return true; + `, eval.ParamString(selector.String())), + true, + false, + ) + + if err != nil { + return values.False, err + } + + if res.Type() == types.Boolean { + return res.(values.Boolean), nil + } + + return values.False, nil +} + +func (doc *HTMLDocument) InputBySelector(ctx context.Context, selector values.String, value core.Value, delay values.Int) (values.Boolean, error) { + valStr := value.String() + + res, err := eval.Eval( + ctx, + doc.client, + fmt.Sprintf(` + var el = document.querySelector(%s); + if (el == null) { + return false; + } + el.focus(); + return true; + `, eval.ParamString(selector.String())), + true, + false, + ) + + if err != nil { + return values.False, err + } + + if res.Type() == types.Boolean && res.(values.Boolean) == values.False { + return values.False, nil + } + + delayMs := time.Duration(delay) + + time.Sleep(delayMs * time.Millisecond) + + for _, ch := range valStr { + for _, ev := range []string{"keyDown", "keyUp"} { + ke := input.NewDispatchKeyEventArgs(ev).SetText(string(ch)) + + if err := doc.client.Input.DispatchKeyEvent(ctx, ke); err != nil { + return values.False, err + } + + time.Sleep(delayMs * time.Millisecond) + } + } + + return values.True, nil +} + +func (doc *HTMLDocument) SelectBySelector(ctx context.Context, selector values.String, value *values.Array) (*values.Array, error) { + res, err := eval.Eval( + ctx, + doc.client, + fmt.Sprintf(` + var element = document.querySelector(%s); + if (element == null) { + return []; + } + var values = %s; + if (element.nodeName.toLowerCase() !== 'select') { + throw new Error('Element is not a element.") + } + + id, err := uuid.NewV4() + + if err != nil { + return nil, err + } + + err = el.client.DOM.SetAttributeValue(ctx, dom.NewSetAttributeValueArgs(el.id.nodeID, attrID, id.String())) + + if err != nil { + return nil, err + } + + res, err := eval.Eval( + ctx, + el.client, + fmt.Sprintf(` + var el = document.querySelector('[%s="%s"]'); + if (el == null) { + return []; + } + var values = %s; + if (el.nodeName.toLowerCase() !== 'select') { + throw new Error('element is not a