Marko Tikvić / webutility

Download as
Browse Code »

Commit 3ad172fb64e351049fdadb2aa02ccfd5abc50fa6

Authored by Marko Tikvić
1 parent 68e590a604
Exists in master

refactored and made public ReadFileLines()

Showing 1 changed file with 26 additions and 17 deletions   Show diff stats
1 package webutility 1 package webutility
2 2
3 import ( 3 import (
4 "database/sql" 4 "database/sql"
5 "encoding/json" 5 "encoding/json"
6 "errors" 6 "errors"
7 "fmt" 7 "fmt"
8 "io" 8 "io"
9 "net/http" 9 "net/http"
10 "os" 10 "os"
11 "strings" 11 "strings"
12 "sync" 12 "sync"
13 "time" 13 "time"
14 14
15 "git.to-net.rs/marko.tikvic/gologger" 15 "git.to-net.rs/marko.tikvic/gologger"
16 ) 16 )
17 17
18 var ( 18 var (
19 mu = &sync.Mutex{} 19 mu = &sync.Mutex{}
20 metadata = make(map[string]Payload) 20 metadata = make(map[string]Payload)
21 21
22 updateQue = make(map[string][]byte) 22 updateQue = make(map[string][]byte)
23 23
24 metadataDB *sql.DB 24 metadataDB *sql.DB
25 activeProject string 25 activeProject string
26 26
27 inited bool 27 inited bool
28 driver string 28 driver string
29 logger *gologger.Logger 29 logger *gologger.Logger
30 ) 30 )
31 31
32 type LangMap map[string]map[string]string 32 type LangMap map[string]map[string]string
33 33
34 type Field struct { 34 type Field struct {
35 Parameter string `json:"param"` 35 Parameter string `json:"param"`
36 Type string `json:"type"` 36 Type string `json:"type"`
37 Visible bool `json:"visible"` 37 Visible bool `json:"visible"`
38 Editable bool `json:"editable"` 38 Editable bool `json:"editable"`
39 } 39 }
40 40
41 type CorrelationField struct { 41 type CorrelationField struct {
42 Result string `json:"result"` 42 Result string `json:"result"`
43 Elements []string `json:"elements"` 43 Elements []string `json:"elements"`
44 Type string `json:"type"` 44 Type string `json:"type"`
45 } 45 }
46 46
47 type Translation struct { 47 type Translation struct {
48 Language string `json:"language"` 48 Language string `json:"language"`
49 FieldsLabels map[string]string `json:"fieldsLabels"` 49 FieldsLabels map[string]string `json:"fieldsLabels"`
50 } 50 }
51 51
52 // output 52 // output
53 type PaginationLinks struct { 53 type PaginationLinks struct {
54 Base string `json:"base"` 54 Base string `json:"base"`
55 Next string `json:"next"` 55 Next string `json:"next"`
56 Prev string `json:"prev"` 56 Prev string `json:"prev"`
57 Self string `json:"self"` 57 Self string `json:"self"`
58 } 58 }
59 59
60 // input 60 // input
61 type PaginationParameters struct { 61 type PaginationParameters struct {
62 URL string `json:"-"` 62 URL string `json:"-"`
63 Offset int64 `json:"offset"` 63 Offset int64 `json:"offset"`
64 Limit int64 `json:"limit"` 64 Limit int64 `json:"limit"`
65 SortBy string `json:"sortBy"` 65 SortBy string `json:"sortBy"`
66 Order string `json:"order"` 66 Order string `json:"order"`
67 } 67 }
68 68
69 // TODO(marko) 69 // TODO(marko)
70 func GetPaginationParameters(req *http.Request) (p PaginationParameters) { 70 func GetPaginationParameters(req *http.Request) (p PaginationParameters) {
71 return p 71 return p
72 } 72 }
73 73
74 // TODO(marko) 74 // TODO(marko)
75 func (p *PaginationParameters) paginationLinks() (links PaginationLinks) { 75 func (p *PaginationParameters) paginationLinks() (links PaginationLinks) {
76 return links 76 return links
77 } 77 }
78 78
79 type Payload struct { 79 type Payload struct {
80 Method string `json:"method"` 80 Method string `json:"method"`
81 Params map[string]string `json:"params"` 81 Params map[string]string `json:"params"`
82 Lang []Translation `json:"lang"` 82 Lang []Translation `json:"lang"`
83 Fields []Field `json:"fields"` 83 Fields []Field `json:"fields"`
84 Correlations []CorrelationField `json:"correlationFields"` 84 Correlations []CorrelationField `json:"correlationFields"`
85 IdField string `json:"idField"` 85 IdField string `json:"idField"`
86 86
87 // Pagination 87 // Pagination
88 Count int64 `json:"count"` 88 Count int64 `json:"count"`
89 Total int64 `json:"total"` 89 Total int64 `json:"total"`
90 Links PaginationLinks `json:"_links"` 90 Links PaginationLinks `json:"_links"`
91 91
92 // Data holds JSON payload. It can't be used for itteration. 92 // Data holds JSON payload. It can't be used for itteration.
93 Data interface{} `json:"data"` 93 Data interface{} `json:"data"`
94 } 94 }
95 95
96 func (p *Payload) SetData(data interface{}) { 96 func (p *Payload) SetData(data interface{}) {
97 p.Data = data 97 p.Data = data
98 } 98 }
99 99
100 func (p *Payload) SetPaginationInfo(count, total int64, params PaginationParameters) { 100 func (p *Payload) SetPaginationInfo(count, total int64, params PaginationParameters) {
101 p.Count = count 101 p.Count = count
102 p.Total = total 102 p.Total = total
103 p.Links = params.paginationLinks() 103 p.Links = params.paginationLinks()
104 } 104 }
105 105
106 // NewPayload returs a payload sceleton for entity described with key. 106 // NewPayload returs a payload sceleton for entity described with key.
107 func NewPayload(r *http.Request, key string) Payload { 107 func NewPayload(r *http.Request, key string) Payload {
108 p := metadata[key] 108 p := metadata[key]
109 p.Method = r.Method + " " + r.RequestURI 109 p.Method = r.Method + " " + r.RequestURI
110 return p 110 return p
111 } 111 }
112 112
113 // DecodeJSON decodes JSON data from r to v. 113 // DecodeJSON decodes JSON data from r to v.
114 // Returns an error if it fails. 114 // Returns an error if it fails.
115 func DecodeJSON(r io.Reader, v interface{}) error { 115 func DecodeJSON(r io.Reader, v interface{}) error {
116 return json.NewDecoder(r).Decode(v) 116 return json.NewDecoder(r).Decode(v)
117 } 117 }
118 118
119 // InitPayloadsMetadata loads all payloads' information into 'metadata' variable. 119 // InitPayloadsMetadata loads all payloads' information into 'metadata' variable.
120 func InitPayloadsMetadata(drv string, db *sql.DB, project string) error { 120 func InitPayloadsMetadata(drv string, db *sql.DB, project string) error {
121 var err error 121 var err error
122 if drv != "ora" && drv != "mysql" { 122 if drv != "ora" && drv != "mysql" {
123 err = errors.New("driver not supported") 123 err = errors.New("driver not supported")
124 return err 124 return err
125 } 125 }
126 126
127 driver = drv 127 driver = drv
128 metadataDB = db 128 metadataDB = db
129 activeProject = project 129 activeProject = project
130 130
131 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB) 131 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB)
132 if err != nil { 132 if err != nil {
133 fmt.Printf("webutility: %s\n", err.Error()) 133 fmt.Printf("webutility: %s\n", err.Error())
134 } 134 }
135 135
136 mu.Lock() 136 mu.Lock()
137 defer mu.Unlock() 137 defer mu.Unlock()
138 err = initMetadata(project) 138 err = initMetadata(project)
139 if err != nil { 139 if err != nil {
140 return err 140 return err
141 } 141 }
142 inited = true 142 inited = true
143 143
144 return nil 144 return nil
145 } 145 }
146 146
147 func EnableHotloading(interval int) { 147 func EnableHotloading(interval int) {
148 if interval > 0 { 148 if interval > 0 {
149 go hotload(interval) 149 go hotload(interval)
150 } 150 }
151 } 151 }
152 152
153 func GetMetadataForAllEntities() map[string]Payload { 153 func GetMetadataForAllEntities() map[string]Payload {
154 return metadata 154 return metadata
155 } 155 }
156 156
157 func GetMetadataForEntity(t string) (Payload, bool) { 157 func GetMetadataForEntity(t string) (Payload, bool) {
158 p, ok := metadata[t] 158 p, ok := metadata[t]
159 return p, ok 159 return p, ok
160 } 160 }
161 161
162 func QueEntityModelUpdate(entityType string, v interface{}) { 162 func QueEntityModelUpdate(entityType string, v interface{}) {
163 updateQue[entityType], _ = json.Marshal(v) 163 updateQue[entityType], _ = json.Marshal(v)
164 } 164 }
165 165
166 func UpdateEntityModels(command string) (total, upd, add int, err error) { 166 func UpdateEntityModels(command string) (total, upd, add int, err error) {
167 if command != "force" && command != "missing" { 167 if command != "force" && command != "missing" {
168 return total, 0, 0, errors.New("webutility: unknown command: " + command) 168 return total, 0, 0, errors.New("webutility: unknown command: " + command)
169 } 169 }
170 170
171 if !inited { 171 if !inited {
172 return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.") 172 return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.")
173 } 173 }
174 174
175 total = len(updateQue) 175 total = len(updateQue)
176 176
177 toUpdate := make([]string, 0) 177 toUpdate := make([]string, 0)
178 toAdd := make([]string, 0) 178 toAdd := make([]string, 0)
179 179
180 for k, _ := range updateQue { 180 for k, _ := range updateQue {
181 if _, exists := metadata[k]; exists { 181 if _, exists := metadata[k]; exists {
182 if command == "force" { 182 if command == "force" {
183 toUpdate = append(toUpdate, k) 183 toUpdate = append(toUpdate, k)
184 } 184 }
185 } else { 185 } else {
186 toAdd = append(toAdd, k) 186 toAdd = append(toAdd, k)
187 } 187 }
188 } 188 }
189 189
190 var uStmt *sql.Stmt 190 var uStmt *sql.Stmt
191 if driver == "ora" { 191 if driver == "ora" {
192 uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2") 192 uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2")
193 if err != nil { 193 if err != nil {
194 logger.Trace(err.Error()) 194 logger.Trace(err.Error())
195 return 195 return
196 } 196 }
197 } else if driver == "mysql" { 197 } else if driver == "mysql" {
198 uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?") 198 uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?")
199 if err != nil { 199 if err != nil {
200 logger.Trace(err.Error()) 200 logger.Trace(err.Error())
201 return 201 return
202 } 202 }
203 } 203 }
204 for _, k := range toUpdate { 204 for _, k := range toUpdate {
205 _, err = uStmt.Exec(string(updateQue[k]), k) 205 _, err = uStmt.Exec(string(updateQue[k]), k)
206 if err != nil { 206 if err != nil {
207 logger.Trace(err.Error()) 207 logger.Trace(err.Error())
208 return 208 return
209 } 209 }
210 upd++ 210 upd++
211 } 211 }
212 212
213 blankPayload, _ := json.Marshal(Payload{}) 213 blankPayload, _ := json.Marshal(Payload{})
214 var iStmt *sql.Stmt 214 var iStmt *sql.Stmt
215 if driver == "ora" { 215 if driver == "ora" {
216 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)") 216 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)")
217 if err != nil { 217 if err != nil {
218 logger.Trace(err.Error()) 218 logger.Trace(err.Error())
219 return 219 return
220 } 220 }
221 } else if driver == "mysql" { 221 } else if driver == "mysql" {
222 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)") 222 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)")
223 if err != nil { 223 if err != nil {
224 logger.Trace(err.Error()) 224 logger.Trace(err.Error())
225 return 225 return
226 } 226 }
227 } 227 }
228 for _, k := range toAdd { 228 for _, k := range toAdd {
229 _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k])) 229 _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k]))
230 if err != nil { 230 if err != nil {
231 logger.Trace(err.Error()) 231 logger.Trace(err.Error())
232 return 232 return
233 } 233 }
234 metadata[k] = Payload{} 234 metadata[k] = Payload{}
235 add++ 235 add++
236 } 236 }
237 237
238 return total, upd, add, nil 238 return total, upd, add, nil
239 } 239 }
240 240
241 func initMetadata(project string) error { 241 func initMetadata(project string) error {
242 rows, err := metadataDB.Query(`select 242 rows, err := metadataDB.Query(`select
243 entity_type, 243 entity_type,
244 metadata 244 metadata
245 from entities 245 from entities
246 where projekat = ` + fmt.Sprintf("'%s'", project)) 246 where projekat = ` + fmt.Sprintf("'%s'", project))
247 if err != nil { 247 if err != nil {
248 return err 248 return err
249 } 249 }
250 defer rows.Close() 250 defer rows.Close()
251 251
252 if len(metadata) > 0 { 252 if len(metadata) > 0 {
253 metadata = nil 253 metadata = nil
254 } 254 }
255 metadata = make(map[string]Payload) 255 metadata = make(map[string]Payload)
256 for rows.Next() { 256 for rows.Next() {
257 var name, load string 257 var name, load string
258 rows.Scan(&name, &load) 258 rows.Scan(&name, &load)
259 259
260 p := Payload{} 260 p := Payload{}
261 err := json.Unmarshal([]byte(load), &p) 261 err := json.Unmarshal([]byte(load), &p)
262 if err != nil { 262 if err != nil {
263 logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load) 263 logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load)
264 } else { 264 } else {
265 metadata[name] = p 265 metadata[name] = p
266 } 266 }
267 } 267 }
268 268
269 return nil 269 return nil
270 } 270 }
271 271
272 func LoadMetadataFromFile(path string) error { 272 func LoadMetadataFromFile(path string) error {
273 lines, err := getFileLines(path) 273 lines, err := ReadFileLines(path)
274 if err != nil { 274 if err != nil {
275 return err 275 return err
276 } 276 }
277 277
278 if len(metadata) > 0 { 278 if len(metadata) > 0 {
279 metadata = nil 279 metadata = nil
280 } 280 }
281 metadata = make(map[string]Payload) 281 metadata = make(map[string]Payload)
282 282
283 var name string 283 var name string
284 for _, l := range lines { 284 for i, l := range lines {
285 if strings.HasPrefix(l, "[") && strings.HasSuffix(l, "]") { 285 l = strings.TrimSpace(l)
286 if len(l) == 0 {
287 continue
288 }
289
290 if strings.HasPrefix(l, "[") {
286 name = strings.Trim(l, "[]") 291 name = strings.Trim(l, "[]")
287 metadata[name] = Payload{ 292 metadata[name] = Payload{
288 Lang: []Translation{ 293 Lang: []Translation{
289 Translation{ 294 Translation{
290 Language: "sr", 295 Language: "sr",
291 FieldsLabels: make(map[string]string), 296 FieldsLabels: make(map[string]string),
292 }, 297 },
293 }, 298 },
294 } 299 }
295 continue 300 continue
296 } 301 }
297 302
303 if name == "" {
304 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [no header][%s]\n", i+1, l)
305 }
306
298 parts := strings.Split(l, ":") 307 parts := strings.Split(l, ":")
299 if len(parts) >= 2 { 308 if len(parts) < 2 {
300 k := strings.TrimSpace(parts[0]) 309 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [invalid format][%s]\n", i+1, l)
301 v := strings.TrimSpace(parts[1]) 310 }
302 if v != "-" { 311
303 metadata[name].Lang[0].FieldsLabels[k] = v 312 k := strings.TrimSpace(parts[0])
304 } 313 v := strings.TrimSpace(parts[1])
314 if v != "-" {
315 metadata[name].Lang[0].FieldsLabels[k] = v
305 } 316 }
306 } 317 }
307 318
308 return nil 319 return nil
309 } 320 }
310 321
311 func getFileLines(path string) ([]string, error) { 322 func ReadFileLines(path string) ([]string, error) {
312 f, err := os.Open(path) 323 f, err := os.Open(path)
313 if err != nil { 324 if err != nil {
314 return nil, err 325 return nil, err
315 } 326 }
316 defer f.Close() 327 defer f.Close()
317 328
318 var s strings.Builder 329 var s strings.Builder
319 for { 330 for {
320 buf := make([]byte, 1024) 331 buf := make([]byte, 1024)
321 _, err := f.Read(buf) 332 n, err := f.Read(buf)
322 if err != nil { 333 if err != nil {
323 if err == io.EOF { 334 if err == io.EOF {
324 s.Write(buf)
325 break 335 break
326 } else {
327 return nil, err
328 } 336 }
337
338 return nil, err
329 } 339 }
330 340
331 s.Write(buf) 341 s.Write(buf[:n])
332 } 342 }
333 343
334 str := s.String() 344 lines := strings.Split(s.String(), "\n")
335 lines := strings.Split(str, "\n")
336 345
337 return lines, nil 346 return lines, nil
338 } 347 }
339 348
340 func hotload(n int) { 349 func hotload(n int) {
341 entityScan := make(map[string]int64) 350 entityScan := make(map[string]int64)
342 firstCheck := true 351 firstCheck := true
343 for { 352 for {
344 time.Sleep(time.Duration(n) * time.Second) 353 time.Sleep(time.Duration(n) * time.Second)
345 rows, err := metadataDB.Query(`select 354 rows, err := metadataDB.Query(`select
346 ora_rowscn, 355 ora_rowscn,
347 entity_type 356 entity_type
348 from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject)) 357 from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject))
349 if err != nil { 358 if err != nil {
350 logger.Log("webutility: hotload failed: %v\n", err) 359 logger.Log("webutility: hotload failed: %v\n", err)
351 time.Sleep(time.Duration(n) * time.Second) 360 time.Sleep(time.Duration(n) * time.Second)
352 continue 361 continue
353 } 362 }
354 363
355 var toRefresh []string 364 var toRefresh []string
356 for rows.Next() { 365 for rows.Next() {
357 var scanID int64 366 var scanID int64
358 var entity string 367 var entity string
359 rows.Scan(&scanID, &entity) 368 rows.Scan(&scanID, &entity)
360 oldID, ok := entityScan[entity] 369 oldID, ok := entityScan[entity]
361 if !ok || oldID != scanID { 370 if !ok || oldID != scanID {
362 entityScan[entity] = scanID 371 entityScan[entity] = scanID
363 toRefresh = append(toRefresh, entity) 372 toRefresh = append(toRefresh, entity)
364 } 373 }
365 } 374 }
366 rows.Close() 375 rows.Close()
367 376
368 if rows.Err() != nil { 377 if rows.Err() != nil {
369 logger.Log("webutility: hotload rset error: %v\n", rows.Err()) 378 logger.Log("webutility: hotload rset error: %v\n", rows.Err())
370 time.Sleep(time.Duration(n) * time.Second) 379 time.Sleep(time.Duration(n) * time.Second)
371 continue 380 continue
372 } 381 }
373 382
374 if len(toRefresh) > 0 && !firstCheck { 383 if len(toRefresh) > 0 && !firstCheck {
375 mu.Lock() 384 mu.Lock()
376 refreshMetadata(toRefresh) 385 refreshMetadata(toRefresh)
377 mu.Unlock() 386 mu.Unlock()
378 } 387 }
379 if firstCheck { 388 if firstCheck {
380 firstCheck = false 389 firstCheck = false
381 } 390 }
382 } 391 }
383 } 392 }
384 393
385 func refreshMetadata(entities []string) { 394 func refreshMetadata(entities []string) {
386 for _, e := range entities { 395 for _, e := range entities {
387 fmt.Printf("refreshing %s\n", e) 396 fmt.Printf("refreshing %s\n", e)
388 rows, err := metadataDB.Query(`select 397 rows, err := metadataDB.Query(`select
389 metadata 398 metadata
390 from entities 399 from entities
391 where projekat = ` + fmt.Sprintf("'%s'", activeProject) + 400 where projekat = ` + fmt.Sprintf("'%s'", activeProject) +
392 ` and entity_type = ` + fmt.Sprintf("'%s'", e)) 401 ` and entity_type = ` + fmt.Sprintf("'%s'", e))
393 402
394 if err != nil { 403 if err != nil {
395 logger.Log("webutility: refresh: prep: %v\n", err) 404 logger.Log("webutility: refresh: prep: %v\n", err)
396 rows.Close() 405 rows.Close()
397 continue 406 continue
398 } 407 }
399 408
400 for rows.Next() { 409 for rows.Next() {
401 var load string 410 var load string
402 rows.Scan(&load) 411 rows.Scan(&load)
403 p := Payload{} 412 p := Payload{}
404 err := json.Unmarshal([]byte(load), &p) 413 err := json.Unmarshal([]byte(load), &p)
405 if err != nil { 414 if err != nil {
406 logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load) 415 logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load)
407 } else { 416 } else {
408 metadata[e] = p 417 metadata[e] = p
409 } 418 }
410 } 419 }
411 rows.Close() 420 rows.Close()
412 } 421 }
413 } 422 }
414 423
415 /* 424 /*
416 func ModifyMetadataForEntity(entityType string, p *Payload) error { 425 func ModifyMetadataForEntity(entityType string, p *Payload) error {
417 md, err := json.Marshal(*p) 426 md, err := json.Marshal(*p)
418 if err != nil { 427 if err != nil {
419 return err 428 return err
420 } 429 }
421 430
422 mu.Lock() 431 mu.Lock()
423 defer mu.Unlock() 432 defer mu.Unlock()
424 _, err = metadataDB.PrepAndExe(`update entities set 433 _, err = metadataDB.PrepAndExe(`update entities set
425 metadata = :1 434 metadata = :1
426 where projekat = :2 435 where projekat = :2
427 and entity_type = :3`, 436 and entity_type = :3`,
428 string(md), 437 string(md),
429 activeProject, 438 activeProject,
430 entityType) 439 entityType)
431 if err != nil { 440 if err != nil {
432 return err 441 return err
433 } 442 }
434 return nil 443 return nil
435 } 444 }
436 445
437 func DeleteEntityModel(entityType string) error { 446 func DeleteEntityModel(entityType string) error {
438 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) 447 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType)
439 if err == nil { 448 if err == nil {
440 mu.Lock() 449 mu.Lock()
441 delete(metadata, entityType) 450 delete(metadata, entityType)
442 mu.Unlock() 451 mu.Unlock()
443 } 452 }