package datatable import ( "context" "crypto/md5" "encoding/json" "fmt" "net/http" "net/url" "regexp" "strconv" "strings" "time" "geeks-accelerator/oss/saas-starter-kit/internal/platform/web" "github.com/pborman/uuid" "github.com/pkg/errors" "gopkg.in/DataDog/dd-trace-go.v1/contrib/go-redis/redis" ) const ( DatatableStateCacheTtl = 120 ) var ( // ErrInvalidColumn occurs when a column can not be mapped correctly. ErrInvalidColumn = errors.New("Invalid column") ) type ( Datatable struct { ctx context.Context w http.ResponseWriter r *http.Request redis *redis.Client fields []DisplayField loadFunc func(ctx context.Context, sorting string, fields []DisplayField) (resp [][]ColumnValue, err error) stateId string req Request resp *Response handleRequest bool sorting []string cacheKey string all [][]ColumnValue loaded bool storeFilteredFieldName string filteredFieldValues []string disableCache bool caseSensitive bool } Request struct { Data string Columns map[int]Column Order map[int]Order Length int Start int Draw int Search Search } Column struct { Name string Data string Orderable bool Searchable bool Search Search } ColumnValue struct { Value string Formatted string } Search struct { Value string IsRegex bool Regexp *regexp.Regexp } Order struct { Column int Dir string } Response struct { AjaxUrl string `json:"ajaxUrl"` Draw int `json:"draw"` RecordsTotal int `json:"recordsTotal"` RecordsFiltered int `json:"recordsFiltered"` Data [][]string `json:"data"` Error string `json:"error"` DisplayFields []DisplayField `json:"displayFields"` } DisplayField struct { Field string `json:"field"` Title string `json:"title"` Visible bool `json:"visible"` Searchable bool `json:"searchable"` Orderable bool `json:"orderable"` Filterable bool `json:"filterable"` AutocompletePath string `json:"autoComplete_path"` FilterItems []FilterOptionItem `json:"filter_items"` FilterPlaceholder string `json:"filter_placeholder"` OrderFields []string `json:"order_fields"` //Type string `json:"type"` } FilterOptionItem struct { Value string `json:"value"` Display string `json:"display"` } ) func (r Request) CacheKey() string { c := Request{ Order: r.Order, // Search : r.Search, } for _, cn := range r.Columns { // these will be applied as a filter cn.Search = Search{} } dat, _ := json.Marshal(c) return fmt.Sprintf("%x", md5.Sum(dat)) } // &columns[0][data]=&columns[0][name]=&columns[0][searchable]=true&columns[0][orderable]=false&columns[0][search][value]=&columns[0][search][regex]=false&columns[1][data]=ts&columns[1][name]=Time&columns[1][searchable]=true&columns[1][orderable]=true&columns[1][search][value]=&columns[1][search][regex]=false&columns[2][data]=level&columns[2][name]=Time&columns[2][searchable]=true&columns[2][orderable]=true&columns[2][search][value]=&columns[2][search][regex]=false&columns[3][data]=msg&columns[3][name]=Time&columns[3][searchable]=true&columns[3][orderable]=true&columns[3][search][value]=&columns[3][search][regex]=false&order[0][column]=0&order[0][dir]=asc&start=0&length=10&search[value]=&search[regex]=false&_=1537426209765 func ParseQueryValues(vals url.Values) (Request, error) { req := Request{ Columns: make(map[int]Column), Order: make(map[int]Order), } var err error for kn, kvs := range vals { pts := strings.Split(kn, "[") switch pts[0] { case "columns": idxStr := strings.Split(pts[1], "]")[0] idx, err := strconv.Atoi(idxStr) if err != nil { return req, err } if _, ok := req.Columns[idx]; !ok { req.Columns[idx] = Column{} } curCol := req.Columns[idx] sn := strings.Split(pts[2], "]")[0] switch sn { case "name": curCol.Name = kvs[0] case "data": curCol.Data = kvs[0] case "orderable": if kvs[0] != "" { curCol.Orderable, err = strconv.ParseBool(kvs[0]) if err != nil { return req, err } } case "searchable": if kvs[0] != "" { curCol.Searchable, err = strconv.ParseBool(kvs[0]) if err != nil { return req, err } } case "search": svn := strings.Split(pts[3], "]")[0] switch svn { case "regex": if kvs[0] != "" { curCol.Search.IsRegex, err = strconv.ParseBool(kvs[0]) if err != nil { return req, err } } case "value": if strings.ToLower(kvs[0]) != "false" { curCol.Search.Value = kvs[0] } default: return req, errors.WithMessagef(ErrInvalidColumn, "Unable to map query Column Search %s for %s", svn, kn) } default: return req, errors.WithMessagef(ErrInvalidColumn, "Unable to map query Column %s for %s", sn, kn) } req.Columns[idx] = curCol case "order": idxStr := strings.Split(pts[1], "]")[0] idx, err := strconv.Atoi(idxStr) if err != nil { return req, err } if _, ok := req.Order[idx]; !ok { req.Order[idx] = Order{} } curOrder := req.Order[idx] sn := strings.Split(pts[2], "]")[0] switch sn { case "dir": curOrder.Dir = kvs[0] case "column": if kvs[0] != "" { curOrder.Column, err = strconv.Atoi(kvs[0]) if err != nil { return req, err } } default: return req, errors.WithMessagef(ErrInvalidColumn, "Unable to map query Order %s for %s", sn, kn) } req.Order[idx] = curOrder case "length": if kvs[0] != "" { req.Length, err = strconv.Atoi(kvs[0]) if err != nil { return req, err } } case "draw": if kvs[0] != "" { req.Draw, err = strconv.Atoi(kvs[0]) if err != nil { return req, err } } case "start": if kvs[0] != "" { req.Start, err = strconv.Atoi(kvs[0]) if err != nil { return req, err } } case "search": sn := strings.Split(pts[1], "]")[0] switch sn { case "value": if strings.ToLower(kvs[0]) != "false" { req.Search.Value = kvs[0] } case "regex": if kvs[0] != "" { req.Search.IsRegex, err = strconv.ParseBool(kvs[0]) if err != nil { return req, err } } default: return req, errors.WithMessagef(ErrInvalidColumn, "Unable to map query Order %s for %s", sn, kn) } } } if req.Search.IsRegex && req.Search.Value != "" { req.Search.Regexp, err = regexp.Compile(req.Search.Value) if err != nil { return req, err } } for idx, col := range req.Columns { if col.Search.IsRegex && col.Search.Value != "" { col.Search.Regexp, err = regexp.Compile(col.Search.Value) if err != nil { return req, err } req.Columns[idx] = col } } return req, nil } func New(ctx context.Context, w http.ResponseWriter, r *http.Request, redisClient *redis.Client, fields []DisplayField, loadFunc func(ctx context.Context, sorting string, fields []DisplayField) (resp [][]ColumnValue, err error)) (dt *Datatable, err error) { dt = &Datatable{ ctx: ctx, w: w, r: r, redis: redisClient, fields: fields, loadFunc: loadFunc, } dt.stateId = r.URL.Query().Get("dtid") if dt.stateId == "" { dt.stateId = uuid.NewRandom().String() } dt.resp = &Response{ Data: [][]string{}, } dt.SetAjaxUrl(r.URL) if web.RequestIsJson(r) { dt.handleRequest = true dt.req, err = ParseQueryValues(r.URL.Query()) if err != nil { return dt, errors.Wrapf(err, "Failed to parse query values") } dt.resp.Draw = dt.req.Draw dt.sorting = []string{} for i := 0; i < len(dt.req.Order); i++ { co := dt.req.Order[i] cn := dt.req.Columns[co.Column] var df DisplayField for _, dc := range dt.fields { if dc.Field == cn.Name { df = dc break } } if df.Field == "" { err = errors.Errorf("Failed to find field for column %s", cn.Name) return dt, err } if len(df.OrderFields) > 0 { for _, of := range df.OrderFields { dt.sorting = append(dt.sorting, fmt.Sprintf("%s %s", of, co.Dir)) } } else { dt.sorting = append(dt.sorting, fmt.Sprintf("%s %s", df.Field, co.Dir)) } } for i := 0; i < len(dt.req.Columns); i++ { cn := dt.req.Columns[i] var cf string for _, dc := range dt.fields { if dc.Field == cn.Name { if dc.Filterable { cn.Searchable = true dt.req.Columns[i] = cn } cf = dc.Field dt.resp.DisplayFields = append(dt.resp.DisplayFields, dc) break } } if cf == "" { err = errors.Errorf("Failed to find field for column %s", cn.Name) return dt, err } } dt.cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(dt.resp.AjaxUrl+dt.req.CacheKey()+dt.stateId))) } else { //for idx, f := range fields { // if f.Filterable && !f.Searchable { // f.Searchable = true // fields[idx] = f // } //} dt.resp.DisplayFields = fields } return dt, nil } func (dt *Datatable) CaseSensitive() { dt.caseSensitive = true } func (dt *Datatable) SetAjaxUrl(u *url.URL) { un, _ := url.Parse(u.String()) qStr := un.Query() qStr.Set("dtid", dt.stateId) // add query to url un.RawQuery = qStr.Encode() if u.IsAbs() { dt.resp.AjaxUrl = un.String() } else { dt.resp.AjaxUrl = un.RequestURI() } } func (dt *Datatable) HasCache() bool { if !dt.handleRequest || dt.disableCache { return false } // @TODO: Need to handle is error better. But when cache is down, the page should still respond, // maybe just need to add logging. cv, _ := dt.redis.WithContext(dt.ctx).Get(dt.cacheKey).Bytes() if len(cv) > 0 { err := json.Unmarshal(cv, &dt.all) if err != nil { // @TODO: Log the error here. } else { dt.loaded = true return true } } return false } func (dt *Datatable) Handled() bool { return dt.handleRequest } func (dt *Datatable) Response() Response { return *dt.resp } func (dt *Datatable) StoreFilteredField(cn string) { dt.storeFilteredFieldName = cn } func (dt *Datatable) GetFilteredFieldValues() []string { return dt.filteredFieldValues } func (dt *Datatable) DisableCache() { dt.disableCache = true } func (dt *Datatable) Render() (rendered bool, err error) { rendered = dt.handleRequest if !rendered { return rendered, nil } if !dt.loaded { sorting := strings.Join(dt.sorting, ",") dt.all, err = dt.loadFunc(dt.ctx, sorting, dt.fields) if err != nil { return rendered, errors.Wrap(err, "Failed to load data") } if !dt.disableCache { dat, err := json.Marshal(dt.all) if err != nil { return rendered, errors.Wrap(err, "Failed to json encode cache response") } err = dt.redis.WithContext(dt.ctx).Set(dt.cacheKey, dat, DatatableStateCacheTtl*time.Second).Err() if err != nil { // @TODO: Log the error here. } } } dt.resp.RecordsTotal = len(dt.all) //fmt.Println("dt.req.Search.Value ", dt.req.Search.Value ) var hasColFilter bool for i := 0; i < len(dt.req.Columns); i++ { cn := dt.req.Columns[i] if !cn.Searchable { continue } if cn.Search.Value != "" { // fmt.Println("col filter on", cn.Name) hasColFilter = true break } } filtered := [][]ColumnValue{} for _, l := range dt.all { var skip bool var oneColAtleastMatches bool for i := 0; i < len(dt.req.Columns); i++ { cn := dt.req.Columns[i] if cn.Name == dt.storeFilteredFieldName { dt.filteredFieldValues = append(dt.filteredFieldValues, l[i].Value) } if !cn.Searchable { // fmt.Println("col ", cn.Name, "is not searchable skipping") continue } if cn.Search.Value != "" { if cn.Search.Regexp != nil { //fmt.Println("col regex", cn.Search.Value, "->>>>", l[i].Value) if !cn.Search.Regexp.MatchString(l[i].Value) { //fmt.Println("-> no match") skip = true if dt.req.Search.Value == "" { // only skip if not full search break } } } else { var match bool if !dt.caseSensitive { match = strings.Contains( strings.ToLower(l[i].Value), strings.ToLower(cn.Search.Value)) } else { match = strings.Contains(l[i].Value, cn.Search.Value) } if !match { //fmt.Println("-> no match") skip = true if dt.req.Search.Value == "" { // only skip if not full search break } } } } if dt.req.Search.Value != "" { if dt.req.Search.Regexp != nil { //fmt.Println("req regex", cn.Search.Value, "->>>>", l[i].Value) if dt.req.Search.Regexp.MatchString(l[i].Value) { // fmt.Println("-> match") oneColAtleastMatches = true if !hasColFilter { // only skip if no column filter break } } } else { if strings.Contains(l[i].Value, dt.req.Search.Value) { // fmt.Println("-> match") oneColAtleastMatches = true if !hasColFilter { // only skip if no column filter break } } } } } if hasColFilter && dt.req.Search.Value != "" { if !skip && oneColAtleastMatches { filtered = append(filtered, l) } } else if hasColFilter { if !skip { filtered = append(filtered, l) } } else if dt.req.Search.Value != "" { if oneColAtleastMatches { filtered = append(filtered, l) } } else { filtered = append(filtered, l) } } dt.resp.RecordsFiltered = len(filtered) for idx, l := range filtered { if dt.req.Start > 0 && idx < dt.req.Start { continue } fl := []string{} for _, lv := range l { if lv.Formatted != "" { fl = append(fl, lv.Formatted) } else { fl = append(fl, lv.Value) } } dt.resp.Data = append(dt.resp.Data, fl) if dt.req.Length > 0 && len(dt.resp.Data) >= dt.req.Length { break } } return rendered, web.RespondJson(dt.ctx, dt.w, dt.resp, http.StatusOK) }