Skip to content

Fix big endian support for parts used by onnxruntime-extensions #192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlekseiNikiforovIBM wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix big endian support for parts used by onnxruntime-extensions #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions blingfireclient.library/inc/BFendian.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License.
*/


#ifndef _BF_ENDIAN_H_
#define _BF_ENDIAN_H_

#include <stdint.h>

#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#include <byteswap.h>
#define from_le16(x) bswap_16(x)
#define from_le32(x) bswap_32(x)
#define from_le64(x) bswap_64(x)
#else // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define from_le16(x) (x)
#define from_le32(x) (x)
#define from_le64(x) (x)
#endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__

template <typename T>
struct byteswap_helper
{
static inline T from_le(T value)
{
return value;
}
};

template <>
struct byteswap_helper<uint16_t>
{
static inline uint16_t from_le(uint16_t value)
{
return from_le16(value);
}
};

template <>
struct byteswap_helper<int16_t>
{
static inline int16_t from_le(int16_t value)
{
return from_le16(value);
}
};

template <>
struct byteswap_helper<uint32_t>
{
static inline uint32_t from_le(uint32_t value)
{
return from_le32(value);
}
};

template <>
struct byteswap_helper<int32_t>
{
static inline int32_t from_le(int32_t value)
{
return from_le32(value);
}
};

template <>
struct byteswap_helper<uint64_t>
{
static inline uint64_t from_le(uint64_t value)
{
return from_le64(value);
}
};

template <>
struct byteswap_helper<int64_t>
{
static inline int64_t from_le(int64_t value)
{
return from_le64(value);
}
};

#endif
33 changes: 21 additions & 12 deletions blingfireclient.library/inc/FAChains_pack_triv.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "FASetImageA.h"
#include "FAUtils_cl.h"
#include "FASecurity.h"
#include "BFendian.h"

namespace BlingFire
{
Expand Down Expand Up @@ -111,29 +112,37 @@ inline const int FAChains_pack_triv::
} else if (sizeof (short) == m_SizeOfValue) {

// get the number of elements
Count = *(const short *)(m_pImage + Offset);
Count = from_le16(*(const short *)(m_pImage + Offset));

if (NULL != pValues && MaxCount >= Count) {
// get the pointer to the encoded elements
const short * pEncodedVals =
(const short *)(m_pImage + Offset + sizeof (short));
// copy elements
for (int i = 0; i < Count; ++i) {
const int Val = pEncodedVals [i];
const int Val = from_le16(pEncodedVals [i]);
pValues [i] = Val;
}
}
} else {

// get the number of elements
Count = *(const int *)(m_pImage + Offset);
Count = from_le32(*(const int *)(m_pImage + Offset));

if (NULL != pValues && MaxCount >= Count) {
// get the pointer to the encoded elements
const int * pEncodedVals =
(const int *)(m_pImage + Offset + sizeof (int));

// copy elements
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
for (int i = 0; i < Count; ++i) {
const int Val = from_le32(pEncodedVals [i]);
pValues [i] = Val;
}
#else // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
memcpy (pValues, pEncodedVals, sizeof (int) * Count);
#endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
}
}

Expand All @@ -153,7 +162,7 @@ inline const int FAChains_pack_triv::
}

// get the number of elements
const int Count = *(const int *)(m_pImage + Offset);
const int Count = from_le32(*(const int *)(m_pImage + Offset));
// get the pointer to the encoded elements
const int * pEncodedVals =
(const int *)(m_pImage + Offset + sizeof (int));
Expand Down Expand Up @@ -194,27 +203,27 @@ inline const int FAChains_pack_triv::
} else if (sizeof (short) == m_SizeOfValue) {

// get the number of elements
Count = *(const short *)(m_pImage + Offset);
Count = from_le16(*(const short *)(m_pImage + Offset));

if (Idx < Count) {
// get the pointer to the encoded elements
const short * pEncodedVals =
(const short *)(m_pImage + Offset + sizeof (short));
// return the value
const int Value = pEncodedVals [Idx];
const int Value = from_le16(pEncodedVals [Idx]);
return Value;
}
} else {

// get the number of elements
Count = *(const int *)(m_pImage + Offset);
Count = from_le32(*(const int *)(m_pImage + Offset));

if (Idx < Count) {
// get the pointer to the encoded elements
const int * pEncodedVals =
(const int *)(m_pImage + Offset + sizeof (int));
// return the value
const int Value = pEncodedVals [Idx];
const int Value = from_le32(pEncodedVals [Idx]);
return Value;
}
}
Expand Down Expand Up @@ -266,7 +275,7 @@ inline const int FAChains_pack_triv::
} else if (sizeof (short) == m_SizeOfValue) {

// get the number of elements
Count = *(const short *)(m_pImage + Offset);
Count = from_le16(*(const short *)(m_pImage + Offset));

// get the pointer to the encoded elements
const short * pEncodedVals = \
Expand All @@ -285,12 +294,12 @@ inline const int FAChains_pack_triv::
if (0 > Idx)
return -1;

const int FoundValue = pEncodedVals [Idx];
const int FoundValue = from_le16(pEncodedVals [Idx]);
return FoundValue;

} else {
// get the number of elements
Count = *(const int *)(m_pImage + Offset);
Count = from_le32(*(const int *)(m_pImage + Offset));

// get the pointer to the encoded elements
const int * pEncodedVals = \
Expand All @@ -302,7 +311,7 @@ inline const int FAChains_pack_triv::
if (0 > Idx)
return -1;

const int FoundValue = pEncodedVals [Idx];
const int FoundValue = from_le32(pEncodedVals [Idx]);
return FoundValue;
}
}
Expand Down
5 changes: 3 additions & 2 deletions blingfireclient.library/inc/FAEncodeUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include "FAConfig.h"
#include "FAFsmConst.h"
#include "BFendian.h"


// Encodes int with variable-length prefix code
Expand Down Expand Up @@ -299,13 +300,13 @@
\
} else if (sizeof (short) == SizeOfValue) { \
\
Value = *(const unsigned short *)(pDump + Offset); \
Value = from_le16(*(const unsigned short *)(pDump + Offset)); \
\
} else { \
\
DebugLogAssert (sizeof (int) == SizeOfValue); \
\
Value = *(const unsigned int *)(pDump + Offset); \
Value = from_le32(*(const unsigned int *)(pDump + Offset)); \
} \
}

Expand Down
7 changes: 4 additions & 3 deletions blingfireclient.library/inc/FAIwMap_pack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "FASetImageA.h"
#include "FAEncodeUtils.h"
#include "FAUtils_cl.h"
#include "BFendian.h"

namespace BlingFire
{
Expand Down Expand Up @@ -76,16 +77,16 @@ inline const int FAIwMap_pack::GetNewIw (const int OldIw) const
}

// get beginning of the interval
const int FromIw = m_pArrFromIw [IntervalIdx];
const int FromIw = from_le32(m_pArrFromIw [IntervalIdx]);
DebugLogAssert (FromIw <= OldIw);

// get pair temporary pointer
const int * pPair = m_pArrToIwOffset + (IntervalIdx << 1);
// get ending of the interval
const int EndIw = *pPair;
const int EndIw = from_le32(*pPair);
pPair++;
// get encoded NewIws' offset for this interval
const int IntervalOffset = *pPair;
const int IntervalOffset = from_le32(*pPair);
DebugLogAssert (0 <= IntervalOffset);

// bigger than the biggest value in this interval
Expand Down
21 changes: 11 additions & 10 deletions blingfireclient.library/inc/FALexTools_t.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "FAWbdConfKeeper.h"
#include "FALimits.h"
#include "FASecurity.h"
#include "BFendian.h"

namespace BlingFire
{
Expand Down Expand Up @@ -168,21 +169,21 @@ inline void FALexTools_t< Ty >::Validate () const
// invalid action
LogAssert (pAct && MinActSize <= ActSize);

const int LeftCx = pAct [0];
const int RightCx = pAct [1];
const int LeftCx = from_le32(pAct [0]);
const int RightCx = from_le32(pAct [1]);
LogAssert (-FALimits::MaxTag <= LeftCx && LeftCx <= FALimits::MaxTag);
LogAssert (-FALimits::MaxTag <= RightCx && RightCx <= FALimits::MaxTag);

int i = ActSize;

// just one tag
if (MinActSize == ActSize && 0 != pAct [MinActSize - 1]) {
if (MinActSize == ActSize && 0 != from_le32(pAct [MinActSize - 1])) {
continue;
// delimiter and fn(s), but no tag
} else if (MinActSize < ActSize && 0 == pAct [MinActSize - 1]) {
} else if (MinActSize < ActSize && 0 == from_le32(pAct [MinActSize - 1])) {
i = MinActSize;
// tag, delimiter and fn(s)
} else if (MinActSize + 1 < ActSize && 0 == pAct [MinActSize]) {
} else if (MinActSize + 1 < ActSize && 0 == from_le32(pAct [MinActSize])) {
i = MinActSize + 1;
} else {
// invalid action
Expand All @@ -191,7 +192,7 @@ inline void FALexTools_t< Ty >::Validate () const

// validate the function id
for (; i < ActSize; ++i) {
const int FnId = pAct [i];
const int FnId = from_le32(pAct [i]);
// bad function id
LogAssert (0 <= FnId && (unsigned) FnId < m_Fn2IniSize);
LogAssert (m_pFn2Ini && 0 <= m_pFn2Ini [FnId]);
Expand Down Expand Up @@ -300,9 +301,9 @@ const int FALexTools_t< Ty >::
const int ActSize = m_pActs->Get (Ow, &pAct);
DebugLogAssert (MinActSize <= ActSize && pAct);

const int LeftCx = pAct [0];
const int RightCx = pAct [1];
const int Tag = pAct [2];
const int LeftCx = from_le32(pAct [0]);
const int RightCx = from_le32(pAct [1]);
const int Tag = from_le32(pAct [2]);

// From position in the data
int FromPos2 = FromPos + LeftCx;
Expand Down Expand Up @@ -349,7 +350,7 @@ const int FALexTools_t< Ty >::
// apply functions, if any
for (; FnIdx < ActSize; ++FnIdx) {

const int FnId = pAct [FnIdx];
const int FnId = from_le32(pAct [FnIdx]);
DebugLogAssert (0 <= FnId && (unsigned) FnId < m_Fn2IniSize);

const int FnIni = m_pFn2Ini [FnId];
Expand Down
Loading