1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
|
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ExplicitNamespaces #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedLists #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE RecordWildCards #-}
-- Implementation of the API. This module is the main point of the program.
module Server (application) where
import Control.Concurrent.STM (TQueue, TVar, atomically,
newTQueue, newTVar,
newTVarIO, readTQueue,
readTVar, writeTQueue,
writeTVar)
import Control.Monad (forever, unless, void,
when)
import Control.Monad.Catch (handle)
import Control.Monad.Extra (ifM, maybeM, unlessM,
whenJust, whenM)
import Control.Monad.IO.Class (MonadIO (liftIO))
import Control.Monad.Logger (LoggingT, NoLoggingT,
logWarnN)
import Control.Monad.Reader (ReaderT, forM)
import Control.Monad.Trans (lift)
import Data.Aeson ((.=))
import qualified Data.Aeson as A
import qualified Data.ByteString.Char8 as C8
import Data.Coerce (coerce)
import Data.Functor ((<&>))
import qualified Data.Map as M
import Data.Pool (Pool)
import Data.Proxy (Proxy (Proxy))
import Data.Swagger (toSchema)
import Data.Text (Text)
import Data.Text.Encoding (decodeUtf8)
import Data.Time (NominalDiffTime,
UTCTime (utctDay),
addUTCTime, diffUTCTime,
getCurrentTime,
nominalDay)
import qualified Data.Vector as V
import Database.Persist
import Database.Persist.Postgresql (SqlBackend,
migrateEnableExtension,
runMigration)
import Fmt ((+|), (|+))
import qualified Network.WebSockets as WS
import Servant (Application,
ServerError (errBody),
err401, err404, serve,
serveDirectoryFileServer,
throwError)
import Servant.API (NoContent (..),
(:<|>) (..))
import Servant.Server (Handler, hoistServer)
import Servant.Swagger (toSwagger)
import API
import qualified GTFS
import Persist
import Server.ControlRoom
import Server.GTFS_RT (gtfsRealtimeServer)
import Server.Util (Service, ServiceM,
runService, sendErrorMsg)
import Yesod (toWaiAppPlain)
import Extrapolation (Extrapolator (..),
LinearExtrapolator (..))
import System.IO.Unsafe
import Conduit (ResourceT)
import Config (ServerConfig (serverConfigAssets, serverConfigZoneinfoPath))
import Data.ByteString (ByteString)
import Data.ByteString.Lazy (toStrict)
import qualified Data.Text as T
import Data.Time.LocalTime.TimeZone.Olson (getTimeZoneSeriesFromOlsonFile)
import Data.Time.LocalTime.TimeZone.Series (TimeZoneSeries)
import Data.UUID (UUID)
import Prometheus
import Prometheus.Metric.GHC
import System.FilePath ((</>))
application :: GTFS.GTFS -> Pool SqlBackend -> ServerConfig -> IO Application
application gtfs dbpool settings = do
doMigration dbpool
metrics <- Metrics
<$> register (gauge (Info "ws_connections" "Number of WS Connections"))
register ghcMetrics
-- TODO: maybe cache these in a TVar, we're not likely to ever need
-- more than one of these
let getTzseries tzname = getTimeZoneSeriesFromOlsonFile
(serverConfigZoneinfoPath settings </> T.unpack tzname)
subscribers <- newTVarIO mempty
pure $ serve (Proxy @CompleteAPI) $ hoistServer (Proxy @CompleteAPI) runService
$ server gtfs getTzseries metrics subscribers dbpool settings
-- databaseMigration :: ConnectionString -> IO ()
doMigration pool = runSql pool $ runMigration $ do
migrateEnableExtension "uuid-ossp"
migrateAll
server :: GTFS.GTFS -> (Text -> IO TimeZoneSeries) -> Metrics -> TVar (M.Map UUID [TQueue (Maybe TrainPing)]) -> Pool SqlBackend -> ServerConfig -> Service CompleteAPI
server gtfs getTzseries Metrics{..} subscribers dbpool settings = handleDebugAPI
:<|> (handleTimetable :<|> handleTimetableStops :<|> handleTrip
:<|> handleRegister :<|> handleTrainPing (throwError err401) :<|> handleWS
:<|> handleSubscribe :<|> handleDebugState :<|> handleDebugTrain
:<|> handleDebugRegister :<|> pure (GTFS.gtfsFile gtfs) :<|> gtfsRealtimeServer gtfs dbpool)
:<|> metrics
:<|> serveDirectoryFileServer (serverConfigAssets settings)
:<|> pure (unsafePerformIO (toWaiAppPlain (ControlRoom gtfs dbpool settings)))
where handleTimetable station maybeDay =
M.filter isLastStop . GTFS.tripsOnDay gtfs <$> liftIO day
where isLastStop = (==) station . GTFS.stationId . GTFS.stopStation . V.last . GTFS.tripStops
day = maybeM (getCurrentTime <&> utctDay) pure (pure maybeDay)
handleTimetableStops day =
pure . A.toJSON . fmap mkJson . M.elems $ GTFS.tripsOnDay gtfs day
where mkJson :: GTFS.Trip GTFS.Deep GTFS.Deep -> A.Value
mkJson GTFS.Trip {..} = A.object
[ "trip" .= tripTripId
, "sequencelength" .= (GTFS.stopSequence . V.last) tripStops
, "stops" .= fmap (\GTFS.Stop{..} -> A.object
[ "departure" .= GTFS.toUTC stopDeparture (GTFS.tzseries gtfs) day
, "arrival" .= GTFS.toUTC stopArrival (GTFS.tzseries gtfs) day
, "station" .= GTFS.stationId stopStation
, "lat" .= GTFS.stationLat stopStation
, "lon" .= GTFS.stationLon stopStation
]) tripStops
]
handleTrip trip = case M.lookup trip (GTFS.trips gtfs) of
Just res -> pure res
Nothing -> throwError err404
handleRegister (ticketId :: UUID) RegisterJson{..} = do
today <- liftIO getCurrentTime <&> utctDay
expires <- liftIO $ getCurrentTime <&> addUTCTime validityPeriod
runSql dbpool $ do
TrackerKey tracker <- insert (Tracker expires False registerAgent)
insert (TrackerTicket (TicketKey ticketId) (TrackerKey tracker))
pure tracker
handleDebugRegister (ticketId :: UUID) = do
expires <- liftIO $ getCurrentTime <&> addUTCTime validityPeriod
runSql dbpool $ do
TrackerKey tracker <- insert (Tracker expires False "debug key")
insert (TrackerTicket (TicketKey ticketId) (TrackerKey tracker))
pure tracker
handleTrainPing onError ping@TrainPing{..} =
let ticketId = trainPingTicket in
isTokenValid dbpool trainPingToken ticketId >>= \case
Nothing -> do
onError
pure Nothing
Just (tracker@Tracker{..}, ticket@Ticket{..}) -> do
runSql dbpool $ do
stations <- selectList [ StopTicket ==. ticketId ] [Asc StopArrival]
>>= mapM (\stop -> do
station <- getJust (stopStation (entityVal stop))
tzseries <- liftIO $ getTzseries (GTFS.tzname (stopArrival (entityVal stop)))
pure (entityVal stop, station, tzseries))
<&> V.fromList
shapePoints <- selectList [ShapePointShape ==. ticketShape] [Asc ShapePointIndex]
<&> (V.fromList . fmap entityVal)
let anchor = extrapolateAnchorFromPing LinearExtrapolator
ticket stations shapePoints ping
insert ping
last <- selectFirst [TrainAnchorTicket ==. trainPingTicket] [Desc TrainAnchorWhen]
-- only insert new estimates if they've actually changed anything
when (fmap (trainAnchorDelay . entityVal) last /= Just (trainAnchorDelay anchor))
$ void $ insert anchor
queues <- liftIO $ atomically $ do
queues <- readTVar subscribers <&> M.lookup (coerce trainPingTicket)
whenJust queues $
mapM_ (\q -> writeTQueue q (Just ping))
pure queues
pure (Just anchor)
handleWS conn = do
liftIO $ WS.forkPingThread conn 30
incGauge metricsWSGauge
handle (\(e :: WS.ConnectionException) -> decGauge metricsWSGauge) $ forever $ do
msg <- liftIO $ WS.receiveData conn
case A.eitherDecode msg of
Left err -> do
logWarnN ("stray websocket message: "+|show msg|+" (could not decode: "+|err|+")")
liftIO $ WS.sendClose conn (C8.pack err)
-- TODO: send a close msg (Nothing) to the subscribed queues? decGauge metricsWSGauge
Right ping ->
-- if invalid token, send a "polite" close request. Note that the client may
-- ignore this and continue sending messages, which will continue to be handled.
liftIO $ handleTrainPing (WS.sendClose conn ("" :: ByteString)) ping >>= \case
Just anchor -> WS.sendTextData conn (A.encode anchor)
Nothing -> pure ()
handleSubscribe (ticketId :: UUID) conn = liftIO $ WS.withPingThread conn 30 (pure ()) $ do
queue <- atomically $ do
queue <- newTQueue
qs <- readTVar subscribers
writeTVar subscribers
$ M.insertWith (<>) ticketId [queue] qs
pure queue
-- send most recent ping, if any (so we won't have to wait for movement)
lastPing <- runSql dbpool $ do
trackers <- getTicketTrackers ticketId
<&> fmap entityKey
selectFirst [TrainPingToken <-. trackers] [Desc TrainPingTimestamp]
<&> fmap entityVal
whenJust lastPing $ \ping ->
WS.sendTextData conn (A.encode lastPing)
handle (\(e :: WS.ConnectionException) -> removeSubscriber queue) $ forever $ do
res <- atomically $ readTQueue queue
case res of
Just ping -> WS.sendTextData conn (A.encode ping)
Nothing -> do
removeSubscriber queue
WS.sendClose conn (C8.pack "train ended")
where removeSubscriber queue = atomically $ do
qs <- readTVar subscribers
writeTVar subscribers
$ M.adjust (filter (/= queue)) ticketId qs
handleDebugState = do
now <- liftIO getCurrentTime
runSql dbpool $ do
tracker <- selectList [TrackerBlocked ==. False, TrackerExpires >=. now] []
pairs <- forM tracker $ \(Entity token@(TrackerKey uuid) _) -> do
entities <- selectList [TrainPingToken ==. token] []
pure (uuid, fmap entityVal entities)
pure (M.fromList pairs)
handleDebugTrain ticketId = do
runSql dbpool $ do
trackers <- getTicketTrackers ticketId
pings <- forM trackers $ \(Entity token _) -> do
selectList [TrainPingToken ==. token] [] <&> fmap entityVal
pure (concat pings)
handleDebugAPI = pure $ toSwagger (Proxy @API)
metrics = exportMetricsAsText <&> (decodeUtf8 . toStrict)
getTicketTrackers :: UUID -> ReaderT SqlBackend (NoLoggingT (ResourceT IO)) [Entity Tracker]
getTicketTrackers ticketId = do
joins <- selectList [TrackerTicketTicket ==. TicketKey ticketId] []
<&> fmap (trackerTicketTracker . entityVal)
selectList [TrackerId <-. joins] []
-- TODO: proper debug logging for expired tokens
isTokenValid :: MonadIO m => Pool SqlBackend -> TrackerId -> TicketId -> m (Maybe (Tracker, Ticket))
isTokenValid dbpool token ticketId = runSql dbpool $ get token >>= \case
Just tracker | not (trackerBlocked tracker) -> do
ifM (hasExpired (trackerExpires tracker))
(pure Nothing)
$ runSql dbpool $ get ticketId
<&> (\case { Nothing -> Nothing; Just ticket -> Just (tracker, ticket) })
_ -> pure Nothing
hasExpired :: MonadIO m => UTCTime -> m Bool
hasExpired limit = do
now <- liftIO getCurrentTime
pure (now > limit)
validityPeriod :: NominalDiffTime
validityPeriod = nominalDay
|