From 9c05b8651827c81abb6dba1dd5d21a9551ab47fc Mon Sep 17 00:00:00 2001 From: Jon Schoning Date: Tue, 21 Jul 2020 19:18:05 -0500 Subject: [PATCH] add ability to set a socks proxy from environment for sending archive requests from the server --- .cf-clearance | 1 - config/settings.yml | 10 +++++----- docker-compose.yml | 7 +++++-- espial.cabal | 6 +++++- package.yaml | 1 + src/Handler/Archive.hs | 15 +++++++++++++-- src/Settings.hs | 13 +++++++------ 7 files changed, 36 insertions(+), 17 deletions(-) delete mode 100644 .cf-clearance diff --git a/.cf-clearance b/.cf-clearance deleted file mode 100644 index 4bed7eb..0000000 --- a/.cf-clearance +++ /dev/null @@ -1 +0,0 @@ -cf_clearance=96f0dfc0741239f896f9443f721c94c8fafa82b0-1595277120-GJDOCYQO diff --git a/config/settings.yml b/config/settings.yml index 2e8f588..f676b1c 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -18,8 +18,8 @@ ip-from-header: "_env:IP_FROM_HEADER:false" # Optional values with the following production defaults. # In development, they default to the inverse. # -# detailed-logging: false -# should-log-all: false +detailed-logging: "_env:DETAILED_LOGGING" # false +should-log-all: "_env:SHOULD_LOG_ALL" # false # reload-templates: false # mutable-static: false # skip-combining: false @@ -37,7 +37,7 @@ database: copyright: Insert copyright statement here #analytics: UA-YOURCODE -# ekg-host: "_env:EKG_HOST:0.0.0.0" -# ekg-port: "_env:EKG_PORT:8000" +archive-socks-proxy-host: "_env:ARCHIVE_SOCKS_PROXY_HOST" +archive-socks-proxy-port: "_env:ARCHIVE_SOCKS_PROXY_PORT" -source-code-uri: "https://github.com/jonschoning/espial/commits/d770116" +source-code-uri: "_env:SOURCE_CODE_URI:https://github.com/jonschoning/espial" diff --git a/docker-compose.yml b/docker-compose.yml index 8c99f2e..c7e367d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,13 @@ services: dockerfile: ../Dockerfile ports: - "3000:3000" - # - "8000:8000" volumes: - '$APPDATA:/app/data' environment: - IP_FROM_HEADER=true - SQLITE_DATABASE=/app/data/espial.sqlite3 - - ekg_datadir=ekg + # - DETAILED_LOGGING=true + # - SHOULD_LOG_ALL=true + # - ARCHIVE_SOCKS_PROXY_HOST=localhost + # - ARCHIVE_SOCKS_PROXY_PORT=8888 + # - SOURCE_CODE_URI=https://github.com/jonschoning/espial diff --git a/espial.cabal b/espial.cabal index cc98474..4f3bb58 100644 --- a/espial.cabal +++ b/espial.cabal @@ -4,7 +4,7 @@ cabal-version: 1.12 -- -- see: https://github.com/sol/hpack -- --- hash: 1e6238118c05016ee28c808f6f736f03c5c719ed69576390d95900d6a5e85cee +-- hash: 7535921358f6f30b353ed1ea8a7bfff26aa471228add3c6392836563ee7fc58d name: espial version: 0.0.8 @@ -136,6 +136,7 @@ library , classy-prelude-conduit >=1.4 && <1.6 , classy-prelude-yesod >=1.4 && <1.6 , conduit >=1.0 && <2.0 + , connection , containers , data-default , directory >=1.1 && <1.4 @@ -208,6 +209,7 @@ executable espial , classy-prelude-conduit >=1.4 && <1.6 , classy-prelude-yesod >=1.4 && <1.6 , conduit >=1.0 && <2.0 + , connection , containers , data-default , directory >=1.1 && <1.4 @@ -277,6 +279,7 @@ executable migration , classy-prelude-conduit >=1.4 && <1.6 , classy-prelude-yesod >=1.4 && <1.6 , conduit >=1.0 && <2.0 + , connection , containers , data-default , directory >=1.1 && <1.4 @@ -351,6 +354,7 @@ test-suite test , classy-prelude-conduit >=1.4 && <1.6 , classy-prelude-yesod >=1.4 && <1.6 , conduit >=1.0 && <2.0 + , connection , containers , data-default , directory >=1.1 && <1.4 diff --git a/package.yaml b/package.yaml index 0478b0b..9ae3989 100644 --- a/package.yaml +++ b/package.yaml @@ -146,6 +146,7 @@ dependencies: # - wai-middleware-metrics - parser-combinators - html-entities +- connection # The library contains all of our application code. The executable # defined below is just a thin wrapper. diff --git a/src/Handler/Archive.hs b/src/Handler/Archive.hs index ec73f09..fa633df 100644 --- a/src/Handler/Archive.hs +++ b/src/Handler/Archive.hs @@ -16,6 +16,7 @@ import qualified Web.FormUrlEncoded as WH import HTMLEntities.Decoder (htmlEncodedText) import Data.Text.Lazy.Builder (toLazyText) import Network.Wai (requestHeaderHost) +import qualified Network.Connection as NC shouldArchiveBookmark :: User -> Key Bookmark -> Handler Bool shouldArchiveBookmark user kbid = do @@ -28,6 +29,14 @@ shouldArchiveBookmark user kbid = do && not (_isArchiveBlacklisted bm) && userArchiveDefault user +getArchiveManager :: Handler Manager +getArchiveManager = do + appSettings <- pure . appSettings =<< getYesod + NH.newTlsManagerWith $ NH.mkManagerSettings def $ + NC.SockSettingsSimple + <$> fmap unpack (appArchiveSocksProxyHost appSettings) + <*> fmap toEnum (appArchiveSocksProxyPort appSettings) + archiveBookmarkUrl :: Key Bookmark -> String -> Handler () archiveBookmarkUrl kbid url = (_fetchArchiveSubmitInfo >>= \case @@ -38,7 +47,8 @@ archiveBookmarkUrl kbid url = userId <- requireAuthId req <- _buildArchiveSubmitRequest submitInfo url -- MM.increment "archive.submit" - res <- liftIO $ NH.httpLbs req =<< NH.getGlobalManager + manager <- getArchiveManager + res <- liftIO $ NH.httpLbs req manager let status = NH.responseStatus res -- MM.increment ("archive.submit_status_" <> (pack.show) (NH.statusCode status)) let updateArchiveUrl = runDB . updateBookmarkArchiveUrl userId kbid . Just @@ -76,7 +86,8 @@ _fetchArchiveSubmitInfo :: Handler (Either String (String , String)) _fetchArchiveSubmitInfo = do -- MM.increment "archive.fetchSubmitId" req <- buildRequest "https://archive.li/" - res <- liftIO $ NH.httpLbs req =<< NH.getGlobalManager + manager <- getArchiveManager + res <- liftIO $ NH.httpLbs req manager -- MM.increment ("archive.fetchSubmitId_status_" <> (pack.show) (NH.statusCode (NH.responseStatus res))) let body = LBS.toStrict (responseBody res) action = _parseSubstring (AP8.string "action=\"") (AP8.notChar '"') body diff --git a/src/Settings.hs b/src/Settings.hs index 8021945..2e4b3cb 100644 --- a/src/Settings.hs +++ b/src/Settings.hs @@ -56,10 +56,11 @@ data AppSettings = AppSettings , appAuthDummyLogin :: Bool -- ^ Indicate if auth dummy login should be enabled. - -- , appEkgHost :: Maybe Text - -- -- ^ Host/interface the ekg server should bind to. - -- , appEkgPort :: Maybe Int - -- -- ^ Port to listen on + , appArchiveSocksProxyHost :: Maybe Text + -- ^ Socks proxy host to use when making archive requests + + , appArchiveSocksProxyPort :: Maybe Int + -- ^ Socks proxy port to use when making archive requests , appSourceCodeUri :: Maybe Text -- ^ Uri to app source code @@ -93,8 +94,8 @@ instance FromJSON AppSettings where appAuthDummyLogin <- o .:? "auth-dummy-login" .!= dev - -- appEkgHost <- o .:? "ekg-host" - -- appEkgPort <- o .:? "ekg-port" + appArchiveSocksProxyHost <- o .:? "archive-socks-proxy-host" + appArchiveSocksProxyPort <- o .:? "archive-socks-proxy-port" appSourceCodeUri <- o .:? "source-code-uri" return AppSettings {..}