FlexRT.Rmd

---
title: "Flex RT with R"
output: html_notebook
---

Bit operations like shift left
```{r}
library("bitops")
```

Reads the main configuration file: Binary strings, images and settings.
```{r}
fwx.size <- file.size(fwx.name)
fwx.handle <- file(fwx.name, "rb")
fwx.data <- readBin(fwx.handle, "raw", fwx.size)
# clean up
close(fwx.handle)
rm("fwx.handle")
```

Functions:
```{r}
# Read 8 bits
d1 <- function(pos) {
  arr1 <- fwx.data[pos+1]
  d <- as.integer(arr1)
  d[1]
}

# Read 16 bits little endian word
d2 <- function(pos) {
  arr2 <- fwx.data[(pos+1):(pos+2)]
  d <- as.integer(arr2)
  d[1] + d[2] * 256
}

# Read 32 bits little endian double word
d4 <- function(pos) {
  arr4 <- fwx.data[(pos+1):(pos+4)]
  d <- as.integer(arr4)
  d[1] + bitShiftL(d[2], 8) + bitShiftL(d[3], 16) + bitShiftL(d[4], 24)
}

# convert a UTF16-LE string (type of strings in FWX file) into UTF-8
get_name_len <- function(name_start, len) {
  name_characters <- list(fwx.data[(name_start+1):(name_start+len*2)])
  name <- iconv(name_characters, "UTF-16LE", "UTF-8")
}

# convert a UTF16-LE string (type of strings in FWX file) into UTF-8
# extract length from the first 16 bit word
get_name <- function(offset) {
  # name length - in 16bits UTF16 Little Endian characters
  len <- d2(offset)
  get_name_len(offset+2, len)
}

# Convert a type code to a type name
data.types <- read.csv('types.csv', stringsAsFactors = FALSE)
type2info <- function(type_code, is_datalink = FALSE) {
  if (is.na(type_code)) {
    type <- data.types[data.types$code==0x00,]
    return(type[!is.na(type$code),])
  }

  # Data link type
  if (is_datalink) {
    if (type_code >= 0 && type_code <= 14) {
      type <- data.types[data.types$datalink_code==type_code,]
      return(type[!is.na(type$code),])
    } else {
      type <- data.types[data.types$code==0x00,]
      return(type[!is.na(type$code),])
    }
  }

  # tag entires type
  # types larger than 0x2000 are arrays
  if (type_code>=0x2000) {
    type_code <- type_code-0x2000
  }
  # extract the correct data type
  type <- data.types[data.types$code==type_code,]
  type <- type[!is.na(type$code),]
  if (dim(type)[1]!=0) {
    return(type)
  } else {
    # create an unknown empty type
    type <- data.types[data.types$code==0x00,]
    return(type[!is.na(type$code),])
  }
}
```

Not sure if it should be 16bit or not, because it always looks the same, the data types are always Little Endian (LSB first)
The first 6 words of 16bit are always the same
```{r}
header.start1 <- d2(0) # 0xbeef
header.start2 <- d2(2) # 0xc
header.start3 <- d2(4) # 0x1
header.start4 <- d2(6) # 0x703
header.start5 <- d2(8) # 0x1
header.start6 <- d2(0xa) # 0x1
```

These two double words seems to say how big is the padding at the last section of the file
It looks like it should be in a 4 bytes alignment (e.g. 0xe2 is actually 0xe4 bytes)
```{r}
header.post_tables_padding1 <- d4(0xc)
header.post_tables_padding2 <- d4(0x10)
```

This is the end of the tables, this way we can tell what is the length of the last object.
```{r}
header.tables_end <- d4(0x14)
```

Most important part, the Table Of Contents (TOC), or the array of arrays. This part tells in which offset every part of the file is. It does not say the length of each part, but you can calculate it by subtracting the next offset from the current.
```{r}
# how many entries in the table of contents
header.toc_entries <- d4(0x18)
# location of table of content, should be the last part in the file
header.toc_offset <- d4(0x1c)
```

I believe this tells the run time which elements to initialize
```{r}
header.init_entries <- d4(0x20)
header.init_offset <- d4(0x24)
```

Just made up some names for these parts. These are 16 bits (double bytes) entries.
```{r}
header.info_entries <- d4(0x28)
header.info_offset <- d4(0x2c)
header.metainfo_entries <- d4(0x30)
header.metainfo_offset <- d4(0x34)
```

How many languages exist in the file and what is their code pages (0x409 - English, 0x407 - German, 0x40D - Hebrew)
```{r}
header.lang_entries <- d4(0x38)
header.lang_offset <- d4(0x3c)
```

Read and store the Table Of Contents (array of arrays)
Entries with 0 are empty
Entries with non-zero contain the offset in the file of the begining of each table
```{r}
# initalize empty TOC array
toc <- array(0,header.toc_entries)
# fill in the TOC with offsets of all arrays in the file
for(i in 1:header.toc_entries) {
  toc[i] <- d4(header.toc_offset + (i-1)*4)
}
```

Read the names of all the tables (arrays) in the file
```{r}
# create an empty data frame for the toc entries
toc.entries <- data.frame()
# iterate thru the TOC to get all table names
for(i in 1:length(toc)) {
  offset <- toc[i]
  if (offset != 0) {
    # how many entries exist in the current table
    entries <- d2(offset)
    # table version? all tables have this type of 0x64, maybe 100 means version 1.00?
    version <- d2(offset+2)
    # parent id? all tables have this 0x01 number
    parent <- d2(offset+4)
    # table id - this number is identical to the index i-1
    id <- d2(offset+6)
    # name of table
    name <- get_name(offset+8)
    
    # create a data.frame row
    next_pos <- dim(toc.entries)[1]+1
    toc.entries[next_pos, 'entries'] <- entries
    toc.entries[next_pos, 'version'] <- version
    toc.entries[next_pos, 'parent'] <- parent
    toc.entries[next_pos, 'id'] <- id
    toc.entries[next_pos, 'name'] <- name
    toc.entries[next_pos, 'offset'] <- offset
    # save the item size
    if (next_pos > 1) {
      toc.entries[next_pos-1, 'item_size'] <- offset - last_offset
    }
    last_offset <- offset
  }
}
# calculate the last item size with the global constant
toc.entries[next_pos, 'item_size'] <- header.tables_end - last_offset
```

Convert Datalink type into a letter
```{r}
# convert number of bits to a single letter
bits2char_arr <- c('X','','','B','W','D','D')
bits2char <- function(bits) {
  return(bits2char_arr[log2(bits)+1])
}
```

Read Datalink which says what is the polling interval of each variable from the S7 server
```{r}
datalink_index <- toc.entries$name == 'DATALINK_READWR'
datalink <- toc.entries[datalink_index,]

# location of offsets of all objects in this table has a constant length
datalinks_offsets <- datalink$offset + 0x34

# create empty data frame to hold all datalinks
dl.entries <- data.frame()

# The number of entries should be the same number as the VAR
for (i in 1:datalink$entries) {
  # location of offset of current item
  datalink_offset <- datalinks_offsets + (i-1)*4
  # relative offset of current item (from beginning of datablock)
  datalink_offset_relative <- d4(datalink_offset)
  # absolute offset of current item (from beginning of file)
  datalink_offset <- datalink_offset_relative + datalinks_offsets + datalink$entries*4
  
  d1a <- d4(k <- datalink_offset) # always 1
  d1b <- d4(k <- k+4) # always 1
  d10 <- d4(k <- k+4) # always 0xa = 10
  d5 <- d2(k <- k+4) # always 5
  d32773 <- d2(k <- k+2) # always 0x8005 = 32773
  acq <- d4(k <- k+2) # Acquisition cycle (in milliseconds)
  d258 <- d2(k <- k+4) # always 0x102 = 256
  d0a <- d2(k <- k+2) # always 0
  d9 <- d4(k <- k+2) # always 9 (for HTTP boolean - it is 6)
  d0b <- d2(k <- k+4) # Always 0
  # see type2info()
  data_type <- d1(k <- k+2) # data_type

  if (d0a == 0) {
    # these two items represent the address of the tag
    database <- d2(k <- k+1)
    database_offset <- d2(k <- k+2)
    k <- k+1
  }

  bit_position <- d1(k <- k+1)
  if (d9 != 2) {
    data24 <- d2(k <- k+1)  
    data26 <- d2(k <- k+2)
    bits <- d2(k <- k+2)
  } else {
    bits <- d1(k <- k+1)
  }
  array_size <- d2(k <- k+2) # array size (usually 1 for a single primitive)
  d7 <- d4(k <- k+2) # always 7
  
  # create a data.frame row
  next_pos <- dim(dl.entries)[1]+1
  dl.entries[next_pos, 'idx'] <- i-1
  dl.entries[next_pos, 'd1a'] <- d1a
  dl.entries[next_pos, 'd1b'] <- d1b
  dl.entries[next_pos, 'd10'] <- d10
  dl.entries[next_pos, 'd5'] <- d5
  dl.entries[next_pos, 'd32773'] <- d32773
  dl.entries[next_pos, 'acq'] <- acq
  dl.entries[next_pos, 'd258'] <- d258
  dl.entries[next_pos, 'd0a'] <- d0a
  dl.entries[next_pos, 'd9'] <- d9
  dl.entries[next_pos, 'd0b'] <- d0b
  dl.entries[next_pos, 'data_type'] <- data_type
  if (d0a == 0) {
    dl.entries[next_pos, 'DB'] <- database
    dl.entries[next_pos, 'DBOFFSET'] <- database_offset
  }
  dl.entries[next_pos, 'bit_position'] <- bit_position
  if (d9 != 2) {
    dl.entries[next_pos, 'd24'] <- data24
    dl.entries[next_pos, 'd26'] <- data26
  }
  dl.entries[next_pos, 'bits'] <- bits
  dl.entries[next_pos, 'array_size'] <- array_size
  dl.entries[next_pos, 'd7'] <- d7

  if (d0a == 0) {
    # calculate the address
    dbchar <- bits2char(bits)
    # extract address
    address <- paste('DB',database,'DB',dbchar,database_offset,sep='')
    if (dbchar == 'X') {
      address <- paste(address,'.',bit_position,sep='')
    }

    dl.entries[next_pos, 'address'] <- address
  }
  
  # save the item size
  if (next_pos > 1) {
    dl.entries[next_pos-1, 'item_size'] <- datalink_offset - last_offset
  }
  last_offset <- datalink_offset
}
# calculate the last item size with the size of the parent table
dl.entries[next_pos, 'item_size'] <- datalink$offset + datalink$item_size - last_offset
```

Read the Tags, this is the list of all variables connected to the server monitoring the system.
```{r}
# create empty data frame to hold all tags
tags.entries <- data.frame()

tags_index <- toc.entries$name == 'VAR'
tags <- toc.entries[tags_index,]
# location of offsets of all objects in this table has a constant length
tags_offsets <- tags$offset + 0x34
# iterate thru all the VARs (tags) and get their names
for(i in 1:tags$entries) {
  # location of offset of current item
  tag_offset <- tags_offsets + (i-1)*4
  # relative offset of current item (from beginning of datablock)
  tag_offset_relative <- d4(tag_offset)
  # absolute offset of current item (from beginning of file)
  tag_offset <- tag_offset_relative + tags_offsets + tags$entries*4

  # String type? Either 3 or 0xb (which is 3+8 - probably some flag)
  type <- d2(tag_offset)
  # name of tag
  name <- get_name(tag_offset+2)
  # must align by 4 bytes the length of the string (in bytes)
  len <- d2(tag_offset+2)
  align <- (len*2) %% 4
  data_offset <- tag_offset+4+len*2+align
  # Some data?
  data_len <- d4(data_offset) # length of current tag, almost always = 2
  data53 <- d2(data_offset+4) # this value is almost always 0x35=53
  idx <- d2(data_offset+6) # this is a running index if data53==53
  dat_offset <- d2(data_offset+8) # this is almost always zero except in cases where data53 != 53
  data80E3 <- d4(data_offset+10) # this value is always = 0x80E3 
  data00 <- d2(data_offset+14) # this value is always = 0
  
  # create a data.frame row
  next_pos <- dim(tags.entries)[1]+1
  tags.entries[next_pos, 'type'] <- type
  tags.entries[next_pos, 'name'] <- name
  tags.entries[next_pos, 'offset'] <- tag_offset
  tags.entries[next_pos, 'data_len'] <- data_len
  tags.entries[next_pos, 'data53'] <- data53
  tags.entries[next_pos, 'idx'] <- idx
  tags.entries[next_pos, 'data_offset'] <- dat_offset
  tags.entries[next_pos, 'data80E3'] <- data80E3
  tags.entries[next_pos, 'data00'] <- data00
  
  if (data_len > 0) {
    data01 <- d2(data_offset+16) # always 0
    data227 <- d2(data_offset+18) # almost always 227 unless it is an array then it is something like 0x8000 or 0x8019

    # see type2name() function for more info about types (INT, Byte...)
    data_type <- d2(data_offset+20) # type code

    array_size <- d2(data_offset+22) # number of sub element, usually = 1 unless it is an array
    
    data_pos <- d2(data_offset+24) # successive numbers with skips of almost always 4
    data02 <- d2(data_offset+26) # this value is always = 0
    data03 <- d2(data_offset+28) # this value is always = 0
    data04 <- d2(data_offset+30) # this value is always = 0
    
    # for data_len 3 - everything shifted by 8 bytes
    if (data_len == 3) {
      data227 <- data02
      data_type <- data03
      array_size <- data04
      data_pos <- d2(data_offset+32)
      data02 <- d2(data_offset+34) # this value is always = 0
      data03 <- d2(data_offset+36) # this value is always = 0
      data04 <- d2(data_offset+38) # this value is always = 0
    }

    tags.entries[next_pos, 'data01'] <- data01
    tags.entries[next_pos, 'data227'] <- data227
    tags.entries[next_pos, 'data_type'] <- data_type
    tags.entries[next_pos, 'array_size'] <- array_size
    tags.entries[next_pos, 'data_pos'] <- data_pos
    tags.entries[next_pos, 'data02'] <- data02
    tags.entries[next_pos, 'data03'] <- data03
    tags.entries[next_pos, 'data04'] <- data04
  }
  
  # save the item size
  if (next_pos > 1) {
    tags.entries[next_pos-1, 'item_size'] <- tag_offset - last_offset
  }
  last_offset <- tag_offset
}
# calculate the last item size with the size of the parent table
tags.entries[next_pos, 'item_size'] <- tags$offset + tags$item_size - last_offset
```