From e76e1e22331c780e037f151befddb9b19f91e0bc Mon Sep 17 00:00:00 2001
From: Johan Walles <johan.walles@gmail.com>
Date: Sun, 10 Nov 2024 22:03:57 +0100
Subject: [PATCH] Add CSV lexer (#1005)

---
 lexers/embedded/csv.xml      | 53 ++++++++++++++++++++++++++++++++++++
 lexers/testdata/csv.actual   |  2 ++
 lexers/testdata/csv.expected | 17 ++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 lexers/embedded/csv.xml
 create mode 100644 lexers/testdata/csv.actual
 create mode 100644 lexers/testdata/csv.expected

diff --git a/lexers/embedded/csv.xml b/lexers/embedded/csv.xml
new file mode 100644
index 0000000..b70c2f8
--- /dev/null
+++ b/lexers/embedded/csv.xml
@@ -0,0 +1,53 @@
+<!--
+Lexer for RFC-4180 compliant CSV subject to the following additions:
+- UTF-8 encoding is accepted (the RFC requires 7-bit ASCII)
+- The line terminator character can be LF or CRLF (the RFC allows CRLF only)
+
+Link to the RFC-4180 specification: https://tools.ietf.org/html/rfc4180
+
+Additions inspired by:
+https://github.com/frictionlessdata/datapackage/issues/204#issuecomment-193242077
+
+Future improvements:
+- Identify non-quoted numbers as LiteralNumber
+- Identify y as an error in "x"y. Currently it's identified as another string
+  literal.
+-->
+
+<lexer>
+    <config>
+        <name>CSV</name>
+        <alias>csv</alias>
+        <filename>*.csv</filename>
+        <mime_type>text/csv</mime_type>
+    </config>
+    <rules>
+        <state name="root">
+            <rule pattern="\r?\n">
+                <token type="Punctuation" />
+            </rule>
+            <rule pattern=",">
+                <token type="Punctuation" />
+            </rule>
+            <rule pattern="&quot;">
+                <token type="LiteralStringDouble" />
+                <push state="escaped" />
+            </rule>
+            <rule pattern="[^\r\n,]+">
+                <token type="LiteralString" />
+            </rule>
+        </state>
+        <state name="escaped">
+            <rule pattern="&quot;&quot;">
+                <token type="LiteralStringEscape"/>
+            </rule>
+            <rule pattern="&quot;">
+                <token type="LiteralStringDouble" />
+                <pop depth="1"/>
+            </rule>
+            <rule pattern="[^&quot;]+">
+                <token type="LiteralStringDouble" />
+            </rule>
+        </state>
+    </rules>
+</lexer>
diff --git a/lexers/testdata/csv.actual b/lexers/testdata/csv.actual
new file mode 100644
index 0000000..f494148
--- /dev/null
+++ b/lexers/testdata/csv.actual
@@ -0,0 +1,2 @@
+hello,"newline:
+quote: "" comma: ,",1,"",""stuff,,1.1
diff --git a/lexers/testdata/csv.expected b/lexers/testdata/csv.expected
new file mode 100644
index 0000000..0a4e12f
--- /dev/null
+++ b/lexers/testdata/csv.expected
@@ -0,0 +1,17 @@
+[
+  {"type":"LiteralString","value":"hello"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralStringDouble","value":"\"newline:\nquote: "},
+  {"type":"LiteralStringEscape","value":"\"\""},
+  {"type":"LiteralStringDouble","value":" comma: ,\""},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralString","value":"1"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralStringDouble","value":"\"\""},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralStringDouble","value":"\"\""},
+  {"type":"LiteralString","value":"stuff"},
+  {"type":"Punctuation","value":",,"},
+  {"type":"LiteralString","value":"1.1"},
+  {"type":"Punctuation","value":"\n"}
+]