sitef.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569
  1. package sitef
  2. import "os"
  3. import "io"
  4. import "strings"
  5. import "fmt"
  6. import "bytes"
  7. import "bufio"
  8. import "strconv"
  9. import "reflect"
  10. import "errors"
  11. import "github.com/beoran/woe/monolog"
  12. // Sitef format for serialization
  13. // Sitef is a simple text format for serializing data to
  14. // It's intent is to be human readable and easy to
  15. // use for multi line text.
  16. // It is quite similar to recfiles, though not compatible because
  17. // in sitef files the first character on the line determines the meaning,
  18. // and there is no backslash escaping.
  19. //
  20. // Sitef is a line based syntax where the first character on the line
  21. // determines the meaning of the line.
  22. // Several lines together form a record.
  23. // A line that starts with # is a comment. There may be no whitespace
  24. // in front of the comment.
  25. // A newline character by itself (that is, an empty line),
  26. // or a - ends a record.
  27. // A plus character, an escape on the previous line or a tab or
  28. // a space continues a value.
  29. // A Continues value gets a newline inserted only when a space or tab was used.
  30. // + supresses the newline.
  31. // Anything else signifies the beginning of the next key.
  32. // % is allowed for special keys for recfile compatibility.
  33. // However % directives are not implemented.
  34. // Keys may not be nested, however, you could use spaces or dots,
  35. // or array indexes to emulate nexted keys.
  36. // A # at the start optionally after whitespace is a comment
  37. //
  38. type Record struct {
  39. dict map[string]string
  40. order []string
  41. }
  42. func NewRecord() (* Record) {
  43. rec := &Record{}
  44. rec.dict = make(map[string]string)
  45. rec.order = make([]string, 0)
  46. return rec
  47. }
  48. func (me * Record) Put(key string, val string) {
  49. me.order = append(me.order, key)
  50. me.dict[key] = val
  51. }
  52. func (me * Record) Putf(key string, format string, values ...interface{}) {
  53. me.Put(key, fmt.Sprintf(format, values...))
  54. monolog.Debug("After putf: %s %v", key, me.order)
  55. }
  56. func (me * Record) PutArrayIndex(key string, index int, value string) {
  57. realkey := fmt.Sprintf("%s[%d]", key, index)
  58. me.Put(realkey, value)
  59. }
  60. func (me * Record) PutArray(key string, values []string) {
  61. for i, value := range values {
  62. me.PutArrayIndex(key, i, value)
  63. }
  64. }
  65. func (me * Record) PutInt(key string, val int) {
  66. me.Putf(key, "%d", val)
  67. }
  68. func (me * Record) PutInt64(key string, val int64) {
  69. me.Putf(key, "%d", val)
  70. }
  71. func (me * Record) PutFloat64(key string, val float64) {
  72. me.Putf(key, "%lf", val)
  73. }
  74. func (me Record) MayGet(key string) (result string, ok bool) {
  75. result, ok = me.dict[key]
  76. return result, ok
  77. }
  78. func (me Record) Get(key string) (result string) {
  79. result= me.dict[key]
  80. return result
  81. }
  82. func (me * Record) GetArrayIndex(key string, i int) (result string) {
  83. realkey := fmt.Sprintf("%s[%d]", key, i)
  84. return me.Get(realkey)
  85. }
  86. func (me Record) Getf(key string, format string,
  87. values ...interface{}) (amount int, ok bool) {
  88. val := me.Get(key)
  89. count, err := fmt.Sscanf(val, format, values...)
  90. if err != nil {
  91. return 0, false
  92. }
  93. return count, true
  94. }
  95. func (me Record) GetInt(key string) (val int, err error) {
  96. i, err := strconv.ParseInt(me.Get(key), 0, 0)
  97. return int(i), err
  98. }
  99. func (me Record) GetIntDefault(key string, def int) (val int) {
  100. i, err := strconv.ParseInt(me.Get(key), 0, 0)
  101. if err != nil {
  102. return def;
  103. }
  104. return int(i);
  105. }
  106. func (me Record) GetFloat(key string) (val float64, error error) {
  107. return strconv.ParseFloat(me.Get(key), 64)
  108. }
  109. func (me * Record) convSimple(typ reflect.Type, val reflect.Value) (res string, err error) {
  110. switch val.Kind() {
  111. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  112. return strconv.FormatInt(val.Int(), 10), nil
  113. case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  114. return strconv.FormatUint(val.Uint(), 10), nil
  115. case reflect.Float32, reflect.Float64:
  116. return strconv.FormatFloat(val.Float(), 'g', -1, val.Type().Bits()), nil
  117. case reflect.String:
  118. return val.String(), nil
  119. case reflect.Bool:
  120. return strconv.FormatBool(val.Bool()), nil
  121. default:
  122. return "", errors.New("Unsupported type")
  123. }
  124. }
  125. func (me * Record) PutValue(key string, value reflect.Value) {
  126. monolog.Debug("PutValue: %s %v", key, value)
  127. stringer, ok := value.Interface().(fmt.Stringer)
  128. if ok {
  129. me.Put(key, stringer.String())
  130. return
  131. }
  132. switch (value.Kind()) {
  133. case reflect.Int, reflect.Int32, reflect.Int64:
  134. me.Putf(key, "%d", value.Int())
  135. case reflect.Uint, reflect.Uint32, reflect.Uint64:
  136. me.Putf(key, "%d", value.Uint())
  137. case reflect.Float32, reflect.Float64:
  138. me.Putf(key, "%f", value.Float())
  139. case reflect.String:
  140. me.Putf(key, "%s", value.String())
  141. case reflect.Struct:
  142. me.PutStruct(key + ".", value.Interface());
  143. default:
  144. me.Put(key, "???")
  145. }
  146. monolog.Debug("Put: key %s value %s, result %v", key, value, me)
  147. }
  148. func (me * Record) PutStruct(prefix string, structure interface {}) {
  149. st := reflect.TypeOf(structure)
  150. vt := reflect.ValueOf(structure)
  151. for i:= 0 ; i < st.NumField() ; i++ {
  152. field := st.Field(i)
  153. key := strings.ToLower(field.Name)
  154. value := vt.Field(i)
  155. me.PutValue(prefix + key, value)
  156. }
  157. }
  158. func (me Record) GetValue(key string, value reflect.Value) (err error){
  159. /*stringer, ok := value.Interface().(fmt.Stringer)
  160. if ok {
  161. me.Gut(key, stringer.String())
  162. return
  163. }*/
  164. monolog.Debug("GetValue: %s %v", key, value)
  165. switch (value.Kind()) {
  166. case reflect.Int, reflect.Int32, reflect.Int64:
  167. value.SetInt(int64(me.GetIntDefault(key, 0)))
  168. case reflect.Uint, reflect.Uint32, reflect.Uint64:
  169. value.SetUint(uint64(me.GetIntDefault(key, 0)))
  170. case reflect.Float32, reflect.Float64:
  171. f, err := me.GetFloat(key)
  172. if (err != nil) {
  173. return err
  174. }
  175. value.SetFloat(f)
  176. case reflect.String:
  177. s, ok := me.MayGet(key)
  178. if (!ok) {
  179. return fmt.Errorf("Could not get string for key %s", key)
  180. }
  181. value.SetString(s)
  182. case reflect.Struct:
  183. me.GetStruct(key + ".", value.Addr().Interface());
  184. default:
  185. monolog.Warning("Don't know what to do with %v", value)
  186. }
  187. return nil
  188. }
  189. func (me Record) GetStruct(prefix string, structure interface {}) {
  190. monolog.Info("GetStruct: structure %v, %v\n", structure,
  191. reflect.TypeOf(structure))
  192. st := reflect.TypeOf(structure).Elem()
  193. vt := reflect.Indirect(reflect.ValueOf(structure))
  194. monolog.Info("GetStruct: type %v value %v\n", st, vt)
  195. for i:= 0 ; i < st.NumField() ; i++ {
  196. field := st.Field(i)
  197. key := prefix + strings.ToLower(field.Name)
  198. value := reflect.Indirect(vt).Field(i)
  199. me.GetValue(key, value)
  200. }
  201. }
  202. type Error struct {
  203. error string
  204. lineno int
  205. }
  206. func (me Error) Error() string {
  207. return fmt.Sprintf("%d: %s", me.Lineno, me.error)
  208. }
  209. func (me Error) Lineno() int {
  210. return me.lineno
  211. }
  212. type ParserState int
  213. const (
  214. PARSER_STATE_INIT ParserState = iota
  215. PARSER_STATE_KEY
  216. PARSER_STATE_VALUE
  217. )
  218. type RecordList []*Record
  219. func ParseReader(read io.Reader) (RecordList, error) {
  220. var records RecordList
  221. record := NewRecord()
  222. var err Error
  223. lineno := 0
  224. scanner := bufio.NewScanner(read)
  225. var key bytes.Buffer
  226. var value bytes.Buffer
  227. for scanner.Scan() {
  228. lineno++
  229. line := scanner.Text()
  230. // End of record?
  231. if (len(line) < 1) || line[0] == '-' {
  232. // Append last record if needed.
  233. if len(key.String()) > 0 {
  234. record.Put(key.String(), value.String())
  235. }
  236. // save the record and make a new one
  237. records = append(records, record)
  238. record = NewRecord()
  239. // comment?
  240. } else if line[0] == '#' {
  241. continue;
  242. // continue value?
  243. } else if line[0] == '\t' || line[0] == ' '|| line[0] == '+' {
  244. /* Add a newline unless + is used */
  245. if (line[0] != '+') {
  246. value.WriteRune('\n')
  247. }
  248. // continue the value, skipping the first character
  249. value.WriteString(line[1:])
  250. // new key
  251. } else if strings.ContainsRune(line, ':') {
  252. // save the previous key/value pair if needed
  253. if len(key.String()) > 0 {
  254. record.Put(key.String(), value.String())
  255. }
  256. key.Reset()
  257. value.Reset()
  258. parts := strings.SplitN(line, ":", 2)
  259. key.WriteString(parts[0])
  260. if len(parts) > 1 {
  261. value.WriteString(parts[1])
  262. }
  263. // Not a key. Be lenient and assume this is a continued value.
  264. } else {
  265. value.WriteString(line)
  266. }
  267. }
  268. // Append last record if needed.
  269. if len(key.String()) > 0 {
  270. record.Put(key.String(), value.String())
  271. }
  272. if (len(record.order) > 0) {
  273. records = append(records, record)
  274. }
  275. if serr := scanner.Err(); serr != nil {
  276. err.lineno = lineno
  277. err.error = serr.Error()
  278. monolog.Error("Sitef parse error: %d %s", lineno, serr.Error)
  279. return records, err
  280. }
  281. return records, nil
  282. }
  283. func ParseFilename(filename string) (RecordList, error) {
  284. file, err := os.Open(filename)
  285. if err != nil {
  286. return nil, err
  287. }
  288. defer file.Close()
  289. return ParseReader(file)
  290. }
  291. func WriteField(writer io.Writer, key string, value string) {
  292. monolog.Debug("WriteField %s:%s", key, value)
  293. replacer := strings.NewReplacer("\n", "\n\t")
  294. writer.Write([]byte(key))
  295. writer.Write([]byte{':'})
  296. writer.Write([]byte(replacer.Replace(value)))
  297. writer.Write([]byte{'\n'})
  298. }
  299. func WriteRecord(writer io.Writer, record Record) {
  300. monolog.Debug("WriteRecord %v", record)
  301. for index := 0 ; index < len(record.order) ; index++ {
  302. key := record.order[index];
  303. value := record.dict[key];
  304. WriteField(writer, key, value);
  305. }
  306. writer.Write([]byte{'-', '-', '-', '-', '\n'})
  307. }
  308. func WriteRecordList(writer io.Writer, records RecordList) {
  309. for _, record := range records {
  310. WriteRecord(writer, *record);
  311. }
  312. }
  313. func SaveRecord(filename string, record Record) (error) {
  314. file, err := os.Create(filename)
  315. if err != nil {
  316. monolog.WriteError(err)
  317. return err
  318. }
  319. defer file.Close()
  320. WriteRecord(file, record)
  321. return nil
  322. }
  323. func SaveRecordList(filename string, records RecordList) (error) {
  324. file, err := os.Create(filename)
  325. if err != nil {
  326. return err
  327. }
  328. defer file.Close()
  329. WriteRecordList(file, records)
  330. return nil
  331. }
  332. /*
  333. func ParseFile(file)
  334. lineno = 0
  335. results = []
  336. errors = []
  337. record = {}
  338. key = nil
  339. value = nil
  340. until file.eof?
  341. lineno += 1
  342. line = file.gets(256)
  343. break if line.nil?
  344. next if line.empty?
  345. // new record
  346. if line[0,2] == '--'
  347. // Store last key used if any.
  348. if key
  349. record[key] = value.chomp
  350. key = nil
  351. end
  352. results << record
  353. record = {}
  354. elsif line[0] == '//'
  355. // Comments start with //
  356. elsif line[0] == ':'
  357. // a key/value pair
  358. key, value = line[1,line.size].split(':', 2)
  359. record[key] = value.chomp
  360. key = value = nil
  361. elsif line[0, 2] == '..'
  362. // end of multiline value
  363. record[key] = value.chomp
  364. key = value = nil
  365. elsif (line[0] == '.') && key.nil?
  366. // Multiline key/value starts here (but is ignored
  367. // until .. is encountered)
  368. key = line[1, line.size]
  369. key.chomp!
  370. value = ""
  371. // multiline value
  372. elsif key
  373. if line[0] == '\\'
  374. // remove any escapes
  375. line.slice!(0)
  376. end
  377. // continue the value
  378. value << line
  379. else
  380. // Not in a key, sntax error.
  381. errors << "//{lineno}: Don't know how to process line"
  382. end
  383. end
  384. // Store last key used if any.
  385. if key
  386. record[key] = value.chomp
  387. end
  388. // store last record
  389. results << record unless record.empty?
  390. return results, errors
  391. end
  392. func load_filename(filename)
  393. results, errors = nil, nil, nil;
  394. file = File.open(filename, 'rt') rescue nil
  395. return nil, ["Could not open //{filename}"] unless file
  396. begin
  397. results, errors = parse_file(file)
  398. ensure
  399. file.close
  400. end
  401. return results, errors
  402. end
  403. // Loads a Sitef fileas obejcts. Uses the ruby_klass atribute to load the object
  404. // If that is missing, uses defklass
  405. func load_objects(filename, defklass=nil)
  406. results, errors = load_filename(filename)
  407. p filename, results, errors
  408. unless errors.nil? || errors.empty?
  409. return nil, errors
  410. end
  411. objres = []
  412. results.each do | result |
  413. klassname = result['ruby_class'] || defklass
  414. return nil unless klassname
  415. klass = klassname.split('::').inject(Kernel) { |klass, name| klass.const_get(name) rescue nil }
  416. return nil unless klass
  417. if klass.respond_to? :from_sitef
  418. objres << klass.from_sitef(result)
  419. else
  420. objres << klass.new(result)
  421. end
  422. end
  423. return objres, errors
  424. end
  425. // Saves a single field to a file in Sitef format.
  426. func save_field(file, key, value)
  427. if value.is_a? String
  428. sval = value.dup
  429. else
  430. sval = value.to_s
  431. end
  432. if sval["\n"]
  433. file.puts(".//{key}\n")
  434. // Escape everything that could be misinterpreted with a \\
  435. sval.gsub!(/\n([\.\-\:\//\\]+)/, "\n\\\\\\1")
  436. sval.gsub!(/\A([\.\-\:\//\\]+)/, "\\\\\\1")
  437. file.printf("%s", sval)
  438. file.printf("\n..\n")
  439. else
  440. file.printf("://{key}://{sval}\n")
  441. end
  442. end
  443. func save_object(file, object, *fields)
  444. save_field(file, :ruby_class, object.class.to_s)
  445. fields.each do | field |
  446. value = object.send(field.to_sym)
  447. save_field(file, field, value)
  448. end
  449. end
  450. func save_record(file, record, *fields)
  451. record.each do | key, value |
  452. next if fields && !fields.empty? && !fields.member?(key)
  453. save_field(file, key, value)
  454. end
  455. end
  456. func save_file(file, records, *fields)
  457. records.each do | record |
  458. if record.is_a? Hash
  459. save_record(file, record, *fields)
  460. else
  461. save_object(file, record, *fields)
  462. end
  463. file.puts("--\n")
  464. end
  465. end
  466. func save_filename(filename, records, *fields)
  467. results , errors = nil, nil
  468. file = File.open(filename, 'wt')
  469. return false, ["Could not open //{filename}"] unless file
  470. begin
  471. save_file(file, records, *fields)
  472. ensure
  473. file.close
  474. end
  475. return true, []
  476. end
  477. end
  478. */