Safe Haskell	None
Language	Haskell2010

Commonmark.Tokens

Synopsis

data Tok = Tok {
- tokType :: !TokType
- tokPos :: !SourcePos
- tokContents :: !Text
}
data TokType
- = Spaces
- | UnicodeSpace
- | LineEnd
- | WordChars
- | Symbol !Char
data SourcePos
tokenize :: String -> Text -> [Tok]
untokenize :: [Tok] -> Text

Documentation

data Tok Source #

Constructors

Tok
Fields tokType :: !TokType tokPos :: !SourcePos tokContents :: !Text

Instances

Instances details

Data Tok Source #
Instance details Defined in Commonmark.Tokens Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Tok -> c Tok gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Tok toConstr :: Tok -> Constr dataTypeOf :: Tok -> DataType dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c Tok) dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok) gmapT :: (forall b. Data b => b -> b) -> Tok -> Tok gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r gmapQ :: (forall d. Data d => d -> u) -> Tok -> [u] gmapQi :: Int -> (forall d. Data d => d -> u) -> Tok -> u gmapM :: Monad m => (forall d. Data d => d -> m d) -> Tok -> m Tok gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Tok -> m Tok gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Tok -> m Tok
Show Tok Source #
Instance details Defined in Commonmark.Tokens Methods showsPrec :: Int -> Tok -> ShowS show :: Tok -> String showList :: [Tok] -> ShowS
Eq Tok Source #
Instance details Defined in Commonmark.Tokens Methods (==) :: Tok -> Tok -> Bool (/=) :: Tok -> Tok -> Bool

data TokType Source #

Constructors

Spaces
UnicodeSpace
LineEnd
WordChars
Symbol !Char

Instances

Instances details

Data TokType Source #
Instance details Defined in Commonmark.Tokens Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> TokType -> c TokType gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c TokType toConstr :: TokType -> Constr dataTypeOf :: TokType -> DataType dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c TokType) dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType) gmapT :: (forall b. Data b => b -> b) -> TokType -> TokType gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> TokType -> r gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> TokType -> r gmapQ :: (forall d. Data d => d -> u) -> TokType -> [u] gmapQi :: Int -> (forall d. Data d => d -> u) -> TokType -> u gmapM :: Monad m => (forall d. Data d => d -> m d) -> TokType -> m TokType gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> TokType -> m TokType gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> TokType -> m TokType
Show TokType Source #
Instance details Defined in Commonmark.Tokens Methods showsPrec :: Int -> TokType -> ShowS show :: TokType -> String showList :: [TokType] -> ShowS
Eq TokType Source #
Instance details Defined in Commonmark.Tokens Methods (==) :: TokType -> TokType -> Bool (/=) :: TokType -> TokType -> Bool
Ord TokType Source #
Instance details Defined in Commonmark.Tokens Methods compare :: TokType -> TokType -> Ordering (<) :: TokType -> TokType -> Bool (<=) :: TokType -> TokType -> Bool (>) :: TokType -> TokType -> Bool (>=) :: TokType -> TokType -> Bool max :: TokType -> TokType -> TokType min :: TokType -> TokType -> TokType

data SourcePos #

Instances

Instances details

Data SourcePos #
Instance details Defined in Text.Parsec.Pos Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> SourcePos -> c SourcePos gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c SourcePos toConstr :: SourcePos -> Constr dataTypeOf :: SourcePos -> DataType dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c SourcePos) dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c SourcePos) gmapT :: (forall b. Data b => b -> b) -> SourcePos -> SourcePos gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> SourcePos -> r gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> SourcePos -> r gmapQ :: (forall d. Data d => d -> u) -> SourcePos -> [u] gmapQi :: Int -> (forall d. Data d => d -> u) -> SourcePos -> u gmapM :: Monad m => (forall d. Data d => d -> m d) -> SourcePos -> m SourcePos gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> SourcePos -> m SourcePos gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> SourcePos -> m SourcePos
Show SourcePos #
Instance details Defined in Text.Parsec.Pos Methods showsPrec :: Int -> SourcePos -> ShowS show :: SourcePos -> String showList :: [SourcePos] -> ShowS
Eq SourcePos #
Instance details Defined in Text.Parsec.Pos Methods (==) :: SourcePos -> SourcePos -> Bool (/=) :: SourcePos -> SourcePos -> Bool
Ord SourcePos #
Instance details Defined in Text.Parsec.Pos Methods compare :: SourcePos -> SourcePos -> Ordering (<) :: SourcePos -> SourcePos -> Bool (<=) :: SourcePos -> SourcePos -> Bool (>) :: SourcePos -> SourcePos -> Bool (>=) :: SourcePos -> SourcePos -> Bool max :: SourcePos -> SourcePos -> SourcePos min :: SourcePos -> SourcePos -> SourcePos

tokenize :: String -> Text -> [Tok] Source #

Convert a Text into a list of Tok. The first parameter species the source name.

untokenize :: [Tok] -> Text Source #

Reverses tokenize. untokenize . tokenize should be the identity.