Skip to content

Commit

Permalink
Merge pull request #20 from ambiata/topic/cleanup
Browse files Browse the repository at this point in the history
Cleanup - better errors, render functions, and indentation.
  • Loading branch information
novemberkilo authored Mar 29, 2017
2 parents 936f27d + 7cc7b94 commit 6beb19f
Show file tree
Hide file tree
Showing 9 changed files with 172 additions and 200 deletions.
146 changes: 43 additions & 103 deletions mafia
Original file line number Diff line number Diff line change
@@ -1,123 +1,63 @@
#!/bin/sh -eu

: ${MAFIA_HOME:=$HOME/.mafia}

fetch_latest () {
if [ -z ${MAFIA_TEST_MODE+x} ]; then
TZ=$(date +"%T")
curl --silent "https://raw.githubusercontent.com/ambiata/mafia/master/script/mafia?$TZ"
else
cat ../script/mafia
fi
}
: ${MAFIA_VERSIONS:=$MAFIA_HOME/versions}

latest_version () {
git ls-remote https://github.com/ambiata/mafia | grep refs/heads/master | cut -f 1
git ls-remote https://github.com/ambiata/mafia | grep refs/heads/master | cut -f 1
}

local_version () {
awk '/^# Version: / { print $3; exit 0; }' $0
build_version() {
MAFIA_VERSION="$1"
MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')
MAFIA_FILE=mafia-$MAFIA_VERSION
MAFIA_PATH=$MAFIA_VERSIONS/$MAFIA_FILE
mkdir -p $MAFIA_VERSIONS
echo "Building $MAFIA_FILE in $MAFIA_TEMP"
git clone https://github.com/ambiata/mafia $MAFIA_TEMP
git --git-dir="$MAFIA_TEMP/.git" --work-tree="$MAFIA_TEMP" reset --hard $MAFIA_VERSION || {
echo "mafia version ($MAFIA_VERSION) could not be found." >&2
exit 1
}
(cd "$MAFIA_TEMP" && ./bin/bootstrap) || {
got=$?
echo "mafia version ($MAFIA_VERSION) could not be built." >&2
exit "$got"
}
chmod +x "$MAFIA_TEMP/.cabal-sandbox/bin/mafia"
# Ensure executable is on same file-system so final mv is atomic.
mv -f "$MAFIA_TEMP/.cabal-sandbox/bin/mafia" "$MAFIA_PATH.$$"
mv "$MAFIA_PATH.$$" "$MAFIA_PATH" || {
rm -f "$MAFIA_PATH.$$"
echo "INFO: mafia version ($MAFIA_VERSION) already exists not overiding," >&2
echo "INFO: this is expected if parallel builds of the same version of" >&2
echo "INFO: mafia occur, we are playing by first in, wins." >&2
exit 0
}
}

run_upgrade () {
MAFIA_TEMP=$(mktemp 2>/dev/null || mktemp -t 'upgrade_mafia')

clean_up () {
rm -f "$MAFIA_TEMP"
}

trap clean_up EXIT

MAFIA_CUR="$0"

if [ -L "$MAFIA_CUR" ]; then
echo 'Refusing to overwrite a symlink; run `upgrade` from the canonical path.' >&2
exit 1
fi

echo "Checking for a new version of mafia ..."
fetch_latest > $MAFIA_TEMP

LATEST_VERSION=$(latest_version)
echo "# Version: $LATEST_VERSION" >> $MAFIA_TEMP

if ! cmp $MAFIA_CUR $MAFIA_TEMP >/dev/null 2>&1; then
mv $MAFIA_TEMP $MAFIA_CUR
chmod +x $MAFIA_CUR
echo "New version found and upgraded. You can now commit it to your git repo."
else
echo "You have latest mafia."
fi
enable_version() {
if [ $# -eq 0 ]; then
MAFIA_VERSION="$(latest_version)"
echo "INFO: No explicit mafia version requested installing latest ($MAFIA_VERSION)." >&2
else
MAFIA_VERSION="$1"
fi
[ -x "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" ] || build_version "$MAFIA_VERSION"
ln -sf "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" "$MAFIA_HOME/versions/mafia"
}

exec_mafia () {
MAFIA_VERSION=$(local_version)

if [ "x$MAFIA_VERSION" = "x" ]; then
# If we can't find the mafia version, then we need to upgrade the script.
run_upgrade
else
MAFIA_BIN=$MAFIA_HOME/bin
MAFIA_FILE=mafia-$MAFIA_VERSION
MAFIA_PATH=$MAFIA_BIN/$MAFIA_FILE

[ -f "$MAFIA_PATH" ] || {
# Create a temporary directory which will be deleted when the script
# terminates. Unfortunately `mktemp` doesn't behave the same on
# Linux and OS/X so we need to try two different approaches.
MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')

# Create a temporary file in MAFIA_BIN so we can do an atomic copy/move dance.
mkdir -p $MAFIA_BIN

clean_up () {
rm -rf "$MAFIA_TEMP"
}

trap clean_up EXIT

echo "Building $MAFIA_FILE in $MAFIA_TEMP"

( cd "$MAFIA_TEMP"

git clone https://github.com/ambiata/mafia
cd mafia

git reset --hard $MAFIA_VERSION

bin/bootstrap ) || exit $?

MAFIA_PATH_TEMP=$(mktemp --tmpdir=$MAFIA_BIN $MAFIA_FILE-XXXXXX 2>/dev/null || TMPDIR=$MAFIA_BIN mktemp -t $MAFIA_FILE)

clean_up_temp () {
clean_up
rm -f "$MAFIA_PATH_TEMP"
}
trap clean_up_temp EXIT

cp "$MAFIA_TEMP/mafia/.cabal-sandbox/bin/mafia" "$MAFIA_PATH_TEMP"
chmod 755 "$MAFIA_PATH_TEMP"
mv "$MAFIA_PATH_TEMP" "$MAFIA_PATH"

clean_up_temp
}

exec $MAFIA_PATH "$@"
fi
[ -x "$MAFIA_HOME/versions/mafia" ] || enable_version
"$MAFIA_HOME/versions/mafia" "$@"
}

#
# The actual start of the script.....
#

if [ $# -gt 0 ]; then
MODE="$1"
else
MODE=""
fi

case "$MODE" in
upgrade) shift; run_upgrade "$@" ;;
case "${1:-}" in
upgrade) shift; enable_version "$@" ;;
*) exec_mafia "$@"
esac
# Version: 7c6993f5ad2ac2a605cbc46cd5a108358b5e8c06
# Version: b4c88e80a2f5539a56e966116743e55413a67bd2
4 changes: 2 additions & 2 deletions master.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[master]
version = 1
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20161027043126-a807489/master-haskell-20161027043126-a807489"
sha1 = "51b26872b0be6c2f248f9bed3721dce09451c3f8"
runner = "s3://ambiata-dispensary-v2/dist/master/master-haskell/linux/x86_64/20170327040840-618a87f/master-haskell-20170327040840-618a87f"
sha1 = "9dab3a57a16a1707fa7920a9d86f1cbca1e3495c"

[build.dist]
GHC_VERSION="7.10.2"
Expand Down
44 changes: 21 additions & 23 deletions src/Regiment/IO.hs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import Control.Monad.IO.Class (liftIO)
import Control.Monad.Trans.Resource (MonadResource (..))
import qualified Control.Monad.Trans.Resource as R

import Data.String (String)

import P

import Regiment.Data
Expand All @@ -28,29 +26,28 @@ import System.IO.Temp (withSystemTempDirectory)
import X.Control.Monad.Trans.Either (EitherT, newEitherT, mapEitherT, runEitherT, firstEitherT)

data RegimentIOError =
RegimentIOReadKeysFailed
| RegimentIOReadPastEOF
| RegimentIONullWrite
| RegimentIOBytestringParseFailed String
| RegimentIOUnpackFailed
| RegimentIOMinOfEmptyVector
| RegimentIOParseError RegimentParseError
RegimentIOParseError RegimentParseError
| RegimentIOMergeError (RegimentMergeError RegimentMergeIOError)
deriving (Eq, Show)

renderRegimentIOError :: RegimentIOError -> Text
renderRegimentIOError _ =
"TODO"
renderRegimentIOError err =
case err of
RegimentIOParseError e ->
renderRegimentParseError e
RegimentIOMergeError e ->
renderRegimentMergeError renderRegimentMergeIOError e

regiment :: InputFile
-> Maybe OutputFile
-> [SortColumn]
-> FormatKind
-> Newline
-> NumColumns
-> Separator
-> MemoryLimit
-> EitherT RegimentIOError IO ()
regiment ::
InputFile
-> Maybe OutputFile
-> [SortColumn]
-> FormatKind
-> Newline
-> NumColumns
-> Separator
-> MemoryLimit
-> EitherT RegimentIOError IO ()
regiment inn out sc f n nc sep m = do
let
fmt =
Expand All @@ -66,9 +63,10 @@ regiment inn out sc f n nc sep m = do
toTempFiles inn (TempDirectory tmp) fmt sc m
firstT RegimentParseMergeError $ merge (TempDirectory tmp) out

merge :: TempDirectory
-> Maybe OutputFile
-> EitherT (RegimentMergeError RegimentMergeIOError) IO ()
merge ::
TempDirectory
-> Maybe OutputFile
-> EitherT (RegimentMergeError RegimentMergeIOError) IO ()
merge (TempDirectory tmp) out = mapEitherT R.runResourceT $ do
fs <- liftIO $ fmap (filter (flip notElem [".", ".."])) $ getDirectoryContents tmp
handles <- mapM (open ReadMode) $ fmap (tmp </>) fs
Expand Down
65 changes: 39 additions & 26 deletions src/Regiment/Parse.hs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
{-# LANGUAGE ScopedTypeVariables #-}
module Regiment.Parse (
RegimentParseError (..)
, renderRegimentParseError
, toTempFiles
, selectSortKeys
, writeCursor
Expand Down Expand Up @@ -34,17 +35,29 @@ import System.IO (IO, IOMode (..))
import qualified System.IO as IO
import System.FilePath ((</>))

import X.Control.Monad.Trans.Either (EitherT, left, newEitherT, runEitherT)
import X.Control.Monad.Trans.Either (EitherT, left, runEitherT, newEitherT, hoistEither)
import qualified X.Data.Vector as Boxed
import qualified X.Data.Vector.Grow as Grow

data RegimentParseError =
RegimentParseKeyNotFound
| RegimentParseIONullWrite
| RegimentParseVectorToKPFailed (Boxed.Vector BS.ByteString)
| RegimentParseVectorToKPFailed Text
| RegimentParseMergeError (RegimentMergeError RegimentMergeIOError)
deriving (Eq, Show)

renderRegimentParseError :: RegimentParseError -> Text
renderRegimentParseError err =
case err of
RegimentParseKeyNotFound ->
"Regiment Parse Error: sort keys not found in vector of parsed fields"
RegimentParseIONullWrite ->
"Regiment Parse Error: write chunk called with a null vector"
RegimentParseVectorToKPFailed t ->
"Regiment Parse Error: Vector could not be converted to KeyedPayload. Reason: " <> t
RegimentParseMergeError e ->
renderRegimentMergeError renderRegimentMergeIOError e

toTempFiles ::
InputFile
-> TempDirectory
Expand Down Expand Up @@ -118,10 +131,11 @@ selectSortKeys bytes parsed sortColumns =
-- returns a vector consisting of keys and payload
False -> Right $ (Boxed.fromList ks) Boxed.++ (Boxed.singleton bytes)

flushVector :: Grow.Grow Boxed.MVector (PrimState IO) (Boxed.Vector BS.ByteString)
-> Int
-> TempDirectory
-> EitherT RegimentParseError IO ()
flushVector ::
Grow.Grow Boxed.MVector (PrimState IO) (Boxed.Vector BS.ByteString)
-> Int
-> TempDirectory
-> EitherT RegimentParseError IO ()
flushVector acc counter (TempDirectory tmp) = do
mv <- Grow.unsafeElems acc
Tim.sort mv
Expand All @@ -132,44 +146,43 @@ flushVector acc counter (TempDirectory tmp) = do
-- done using 'v'
Grow.clear acc

writeChunk :: IO.Handle
-> Boxed.Vector (Boxed.Vector BS.ByteString)
-> EitherT RegimentParseError IO ()
writeChunk ::
IO.Handle
-> Boxed.Vector (Boxed.Vector BS.ByteString)
-> EitherT RegimentParseError IO ()
writeChunk h vs =
case Boxed.uncons vs of
Nothing -> left RegimentParseIONullWrite
Just (bs, tl) -> do
let
maybeKp = vecToKP bs
case maybeKp of
Nothing -> left $ RegimentParseVectorToKPFailed bs
Just kp -> do
liftIO $ writeCursor h kp
if Boxed.null tl
then return ()
else writeChunk h tl
kp <- hoistEither $ vecToKP bs
liftIO $ writeCursor h kp
if Boxed.null tl
then return ()
else writeChunk h tl

writeCursor :: IO.Handle -> KeyedPayload -> IO ()
writeCursor h kp = do
liftIO $ Builder.hPutBuilder h (Builder.int32LE . sizeKeyedPayload $ kp)
liftIO $ Builder.hPutBuilder h (bKeyedPayload kp)


vecToKP :: Boxed.Vector BS.ByteString -> Maybe KeyedPayload
vecToKP :: Boxed.Vector BS.ByteString -> Either RegimentParseError KeyedPayload
vecToKP vbs =
-- expected format of Vector
-- [k_1,k_2,...,k_n,payload] where k_i are sortkeys
-- expect at least one sortkey
-- [k_1,k_2,...,k_n,payload] where k_i are sort keys
-- expect n >= 1, thus length vbs >= 2
let
l = Boxed.length vbs
in
if l > 1
then
case l of
0 ->
Left . RegimentParseVectorToKPFailed $ "Empty vector"
1 ->
Left . RegimentParseVectorToKPFailed $ "Singleton vector: " <> (T.pack . show $ Boxed.head vbs)
_ ->
let
p = Boxed.last vbs
sks = Key <$> Boxed.take (l-1) vbs
in
Just $ KeyedPayload sks p
else
Nothing
Right $ KeyedPayload sks p

3 changes: 0 additions & 3 deletions src/Regiment/Serial.hs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,3 @@ getKeyedPayload = do
KeyedPayload
<$> getKeys (bcount - 1)
<*> getSizedPrefixedBytes



Loading

0 comments on commit 6beb19f

Please sign in to comment.