thyphoon/PBMap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better thread/download management

Browse Source
This commit is contained in:
djes
2017-06-15 12:24:23 +02:00
parent b11176a46f
commit 55bbe10378
1 changed files with 89 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
PBMap.pb
View File

@@ -128,6 +128,7 @@ Module PBMap
Download.i Download.i
Time.i Time.i
Size.i Size.i
Mutex.i
EndStructure EndStructure
Structure BoundingBox Structure BoundingBox
@@ -155,6 +156,7 @@ Module PBMap
Structure ImgMemCach Structure ImgMemCach
nImage.i nImage.i
Size.i
*Tile.Tile *Tile.Tile
*TimeStackPtr *TimeStackPtr
Alpha.i Alpha.i
@@ -298,7 +300,6 @@ Module PBMap
DownloadSlots.i ; Actual nb of used download slots DownloadSlots.i ; Actual nb of used download slots
DownloadSlotsMutex.i ; To be sure that only one thread at a time can access to the DownloadSlots var DownloadSlotsMutex.i ; To be sure that only one thread at a time can access to the DownloadSlots var
List TracksList.Tracks() ; To display a GPX track List TracksList.Tracks() ; To display a GPX track
List Markers.Marker() ; To diplay marker List Markers.Marker() ; To diplay marker
EditMarker.l EditMarker.l
@@ -1056,7 +1057,7 @@ Module PBMap
Procedure MemoryCacheManagement() Procedure MemoryCacheManagement()
; MemoryCache access management ; MemoryCache access management
LockMutex(PBMap\MemoryCacheAccessNBMutex) LockMutex(PBMap\MemoryCacheAccessNBMutex)
; If MemoryCache is not being used by a download thread ; If MemoryCache is not being used by any download thread
If PBMap\MemoryCacheAccessNB = 0 If PBMap\MemoryCacheAccessNB = 0
PBMap\MemoryCacheAccessNB = -1 ; Not really useful as the download thread are now blocked by the mutex, and this procedure is synchronous PBMap\MemoryCacheAccessNB = -1 ; Not really useful as the download thread are now blocked by the mutex, and this procedure is synchronous
;UnlockMutex(PBMap\MemoryCacheAccessNBMutex) ;UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
@@ -1109,7 +1110,7 @@ Module PBMap
Protected nImage.i, LifeTime.i, MaxLifeTime.i Protected nImage.i, LifeTime.i, MaxLifeTime.i
MaxLifeTime.i = PBMap\Options\TileLifetime MaxLifeTime.i = PBMap\Options\TileLifetime
If FileSize(CacheFile) > 0 ; <> -1 If FileSize(CacheFile) > 0 ; <> -1
; Manage tile file lifetime ; Manage tile file lifetime
If MaxLifeTime <> -1 If MaxLifeTime <> -1
LifeTime = Date() - GetFileDate(CacheFile, #PB_Date_Modified) ; There's a bug with #PB_Date_Created LifeTime = Date() - GetFileDate(CacheFile, #PB_Date_Modified) ; There's a bug with #PB_Date_Created
If LifeTime > MaxLifeTime If LifeTime > MaxLifeTime
@@ -1164,62 +1165,74 @@ Module PBMap
;-*** These are threaded ;-*** These are threaded
Threaded Progress = 0, Size = 0, Quit = #False Threaded Progress = 0, Quit = #False
Procedure GetImageThread(*Tile.Tile) Procedure GetImageThread(*Tile.Tile)
MyDebug("Thread starting for image " + *Tile\CacheFile + "(" + *Tile\key + ")", 5) MyDebug("Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " starting for image " + *Tile\CacheFile, 5)
;*** Waits for a free download slot
LockMutex(PBMap\DownloadSlotsMutex) LockMutex(PBMap\DownloadSlotsMutex)
While PBMap\DownloadSlots >= PBMap\Options\MaxDownloadSlots ; Is there's no free download slot, abort
If PBMap\DownloadSlots >= PBMap\Options\MaxDownloadSlots
MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " canceled because no free download slot.", 5)
*Tile\Size = 0 ; \Size = 0 signals that the download has failed
UnlockMutex(PBMap\DownloadSlotsMutex) UnlockMutex(PBMap\DownloadSlotsMutex)
If ElapsedMilliseconds() - *Tile\Time > 10000 PostEvent(#PB_Event_Gadget, PBMap\Window, PBmap\Gadget, #PB_MAP_TILE_CLEANUP, *Tile) ; To free memory outside the thread
*Tile\Size = 0 ; \Size = 0 signals that the download has failed ProcedureReturn
MyDebug(" Thread for image " + *Tile\CacheFile + " canceled after 10 seconds waiting for a slot.", 5) EndIf
PostEvent(#PB_Event_Gadget, PBMap\Window, PBmap\Gadget, #PB_MAP_TILE_CLEANUP, *Tile) ; To free memory outside the thread
ProcedureReturn #False
EndIf
MyDebug(" Thread for image " + *Tile\CacheFile + " waiting a download slot", 5)
Delay(20)
LockMutex(PBMap\DownloadSlotsMutex)
Wend
PBMap\DownloadSlots + 1 PBMap\DownloadSlots + 1
UnlockMutex(PBMap\DownloadSlotsMutex) UnlockMutex(PBMap\DownloadSlotsMutex)
;*** ; *** Waits for a free download slot
; LockMutex(PBMap\DownloadSlotsMutex)
; While PBMap\DownloadSlots >= PBMap\Options\MaxDownloadSlots
; UnlockMutex(PBMap\DownloadSlotsMutex)
; If ElapsedMilliseconds() - *Tile\Time > 10000
; *Tile\Size = 0 ; \Size = 0 signals that the download has failed
; MyDebug(" Thread for image " + *Tile\CacheFile + " canceled after 10 seconds waiting for a slot.", 5)
; PostEvent(#PB_Event_Gadget, PBMap\Window, PBmap\Gadget, #PB_MAP_TILE_CLEANUP, *Tile) ; To free memory outside the thread
; ProcedureReturn #False
; EndIf
; MyDebug(" Thread for image " + *Tile\CacheFile + " waiting a download slot", 5)
; Delay(20)
; LockMutex(PBMap\DownloadSlotsMutex)
; Wend
; PBMap\DownloadSlots + 1
; UnlockMutex(PBMap\DownloadSlotsMutex)
; ***
; MemoryCache access management ; MemoryCache access management
LockMutex(PBMap\MemoryCacheAccessNBMutex) LockMutex(PBMap\MemoryCacheAccessNBMutex)
; If MemoryCache is currently being cleaned, wait ; If MemoryCache is currently being cleaned, abort
While PBMap\MemoryCacheAccessNB = -1 If PBMap\MemoryCacheAccessNB = -1
MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " canceled because of cleaning.", 5)
*Tile\Size = 0 ; \Size = 0 signals that the download has failed
UnlockMutex(PBMap\MemoryCacheAccessNBMutex) UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
Delay(20) PostEvent(#PB_Event_Gadget, PBMap\Window, PBmap\Gadget, #PB_MAP_TILE_CLEANUP, *Tile) ; To free memory outside the thread
LockMutex(PBMap\MemoryCacheAccessNBMutex) ProcedureReturn
Wend EndIf
; We're accessing MemoryCache ; We're accessing MemoryCache
PBMap\MemoryCacheAccessNB + 1 PBMap\MemoryCacheAccessNB + 1
UnlockMutex(PBMap\MemoryCacheAccessNBMutex) UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
*Tile\Download = ReceiveHTTPFile(*Tile\URL, *Tile\CacheFile, #PB_HTTP_Asynchronous) *Tile\Download = ReceiveHTTPFile(*Tile\URL, *Tile\CacheFile, #PB_HTTP_Asynchronous)
If *Tile\Download If *Tile\Download
Repeat Repeat
;If PBMap\MemoryCacheManagement = #False ; Wait until cache cleaning is done ;TODO
Progress = HTTPProgress(*Tile\Download) Progress = HTTPProgress(*Tile\Download)
Select Progress Select Progress
Case #PB_Http_Success Case #PB_Http_Success
*Tile\Size = FinishHTTP(*Tile\Download) ; \Size signals that the download is OK *Tile\Size = FinishHTTP(*Tile\Download) ; \Size signals that the download is OK
MyDebug(" Thread for image " + *Tile\CacheFile + " finished. Size : " + Str(Size), 5) MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " finished. Size : " + Str(*Tile\Size), 5)
Quit = #True Quit = #True
Case #PB_Http_Failed Case #PB_Http_Failed
FinishHTTP(*Tile\Download) FinishHTTP(*Tile\Download)
*Tile\Size = 0 ; \Size = 0 signals that the download has failed *Tile\Size = 0 ; \Size = 0 signals that the download has failed
MyDebug(" Thread for image " + *Tile\CacheFile + " failed.", 5) MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " failed.", 5)
Quit = #True Quit = #True
Case #PB_Http_Aborted Case #PB_Http_Aborted
FinishHTTP(*Tile\Download) FinishHTTP(*Tile\Download)
*Tile\Size = 0 ; \Size = 0 signals that the download has failed *Tile\Size = 0 ; \Size = 0 signals that the download has failed
MyDebug(" Thread for image " + *Tile\CacheFile + " aborted.", 5) MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " aborted.", 5)
Quit = #True Quit = #True
Default Default
MyDebug(" Thread for image " + *Tile\CacheFile + " downloading " + Str(Progress) + " bytes", 5) MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " downloading " + Str(Progress) + " bytes", 5)
If ElapsedMilliseconds() - *Tile\Time > 10000 If ElapsedMilliseconds() - *Tile\Time > 10000
MyDebug(" Thread for image " + *Tile\CacheFile + " canceled after 10 seconds.", 5) MyDebug(" Thread nb " + Str(*Tile\GetImageThread) + " " + *Tile\key + " for image " + *Tile\CacheFile + " canceled after 10 seconds.", 5)
AbortHTTP(*Tile\Download) AbortHTTP(*Tile\Download)
EndIf EndIf
EndSelect EndSelect
@@ -1240,6 +1253,8 @@ Module PBMap
;-*** ;-***
Procedure.i GetTile(key.s, URL.s, CacheFile.s) Procedure.i GetTile(key.s, URL.s, CacheFile.s)
; MemoryCache access management
LockMutex(PBMap\MemoryCacheAccessNBMutex)
; Try to find the tile in memory cache ; Try to find the tile in memory cache
Protected *timg.ImgMemCach = FindMapElement(PBMap\MemCache\Images(), key) Protected *timg.ImgMemCach = FindMapElement(PBMap\MemCache\Images(), key)
If *timg If *timg
@@ -1248,16 +1263,16 @@ Module PBMap
If *timg\nImage If *timg\nImage
; Yes, returns the image's nb ; Yes, returns the image's nb
MyDebug(" as image " + *timg\nImage, 4) MyDebug(" as image " + *timg\nImage, 4)
*timg\Tile = -1
; *** Cache management ; *** Cache management
; Retrieves the image in the time stack, push it to the end (to say it's the lastly used) ; Retrieves the image in the time stack, push it to the end (to say it's the lastly used)
ChangeCurrentElement(PBMap\MemCache\ImagesTimeStack(), *timg\TimeStackPtr) ChangeCurrentElement(PBMap\MemCache\ImagesTimeStack(), *timg\TimeStackPtr)
MoveElement(PBMap\MemCache\ImagesTimeStack(), #PB_List_Last) MoveElement(PBMap\MemCache\ImagesTimeStack(), #PB_List_Last)
; *timg\TimeStackPtr = LastElement(PBMap\MemCache\ImagesTimeStack()) ; *timg\TimeStackPtr = LastElement(PBMap\MemCache\ImagesTimeStack())
; *** ; ***
UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
ProcedureReturn *timg ProcedureReturn *timg
Else Else
; No, will load it below ; No, try to load it from HD (see below)
MyDebug(" but not the image.", 4) MyDebug(" but not the image.", 4)
EndIf EndIf
Else Else
@@ -1265,6 +1280,7 @@ Module PBMap
*timg = AddMapElement(PBMap\MemCache\Images(), key) *timg = AddMapElement(PBMap\MemCache\Images(), key)
If *timg = 0 If *timg = 0
MyDebug(" Can't add a new cache element.", 4) MyDebug(" Can't add a new cache element.", 4)
UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
ProcedureReturn #False ProcedureReturn #False
EndIf EndIf
; add a new time stack element at the End ; add a new time stack element at the End
@@ -1274,55 +1290,56 @@ Module PBMap
If *timg\TimeStackPtr = 0 If *timg\TimeStackPtr = 0
MyDebug(" Can't add a new time stack element.", 4) MyDebug(" Can't add a new time stack element.", 4)
DeleteMapElement(PBMap\MemCache\Images()) DeleteMapElement(PBMap\MemCache\Images())
UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
ProcedureReturn #False ProcedureReturn #False
EndIf EndIf
; Associates the time stack element to the cache element ; Associates the time stack element to the cache element
PBMap\MemCache\ImagesTimeStack()\MapKey = MapKey(PBMap\MemCache\Images()) PBMap\MemCache\ImagesTimeStack()\MapKey = MapKey(PBMap\MemCache\Images())
MyDebug("Key : " + key + " added in memory cache", 4) MyDebug("Key : " + key + " added in memory cache", 4)
; ***
EndIf EndIf
; Is this tile not loading ? ; If there's no active download thread for this tile
If *timg\Tile <= 0 If *timg\Tile <= 0
; Is the file image on HDD ? ; Try To load it from HD
*timg\nImage = GetTileFromHDD(CacheFile.s) *timg\nImage = GetTileFromHDD(CacheFile.s)
If *timg\nImage If *timg\nImage
; Image found and loaded from HDD ; Image found and loaded from HDD
*timg\Alpha = 0 *timg\Alpha = 0
*timg\Tile = -1 UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
ProcedureReturn *timg ProcedureReturn *timg
EndIf EndIf
EndIf ; If GetTileFromHDD failed, will load it (again?) from the web (see below)
; If there's not this tile in memory and no web loading thread If *timg\nImage = 0
If *timg\Tile = 0 ; Launch a new web loading thread
; Launch a new one If PBMap\ThreadsNB < PBMap\Options\MaxThreads
If PBMap\ThreadsNB < PBMap\Options\MaxThreads Protected *NewTile.Tile = AllocateMemory(SizeOf(Tile))
Protected *NewTile.Tile = AllocateMemory(SizeOf(Tile)) If *NewTile
If *NewTile With *NewTile
With *NewTile ; New tile parameters
; New tile parameters \key = key
\key = key \URL = URL
\URL = URL \CacheFile = CacheFile
\CacheFile = CacheFile \nImage = 0
\nImage = 0 \Time = ElapsedMilliseconds()
\Time = ElapsedMilliseconds() \GetImageThread = CreateThread(@GetImageThread(), *NewTile)
\GetImageThread = CreateThread(@GetImageThread(), *NewTile) If \GetImageThread
If \GetImageThread *timg\Tile = *NewTile ; There's now a loading thread
*timg\Tile = *NewTile ; There's now a loading thread *timg\Alpha = 0
*timg\Alpha = 0 MyDebug(" Creating get image thread nb " + Str(\GetImageThread) + " to get " + CacheFile + " (key = " + key, 3)
MyDebug(" Creating get image thread nb " + Str(\GetImageThread) + " to get " + CacheFile, 3) PBMap\ThreadsNB + 1
PBMap\ThreadsNB + 1 Else
Else MyDebug(" Can't create get image thread to get " + CacheFile, 3)
MyDebug(" Can't create get image thread to get " + CacheFile, 3) FreeMemory(*NewTile)
FreeMemory(*NewTile) EndIf
EndIf EndWith
EndWith Else
Else MyDebug(" Error, can't allocate memory for a new tile loading thread", 3)
MyDebug(" Error, can't allocate memory for a new tile loading thread", 3) EndIf
Else
MyDebug(" Error, maximum threads nb reached", 3)
EndIf EndIf
Else
MyDebug(" Error, maximum threads nb reached", 3)
EndIf EndIf
EndIf EndIf
UnlockMutex(PBMap\MemoryCacheAccessNBMutex)
ProcedureReturn #False ProcedureReturn #False
EndProcedure EndProcedure
@@ -2488,12 +2505,14 @@ Module PBMap
*Tile = EventData() *Tile = EventData()
key = *Tile\key key = *Tile\key
*Tile\Download = 0 *Tile\Download = 0
PBMap\MemCache\Images(key)\Tile = *Tile\Size
If *Tile\Size ; <> 0 If *Tile\Size ; <> 0
; Web loading thread has finished successfully
FreeMemory(*Tile) ; Frees the data needed for the thread (*tile=PBMap\MemCache\Images(key)\Tile) FreeMemory(*Tile) ; Frees the data needed for the thread (*tile=PBMap\MemCache\Images(key)\Tile)
PBMap\MemCache\Images(key)\Tile = -1 ; Clears the data ptr, and says that the web loading thread has finished successfully PBMap\MemCache\Images(key)\Tile = -1
Else Else
FreeMemory(*Tile) ; Frees the data needed for the thread (*tile=PBMap\MemCache\Images(key)\Tile) FreeMemory(*Tile) ; Frees the data needed for the thread (*tile=PBMap\MemCache\Images(key)\Tile)
PBMap\MemCache\Images(key)\Tile = 0 ; Clears the data ptr, and says that the web loading thread has finished unsuccessfully PBMap\MemCache\Images(key)\Tile = 0
EndIf EndIf
PBMap\ThreadsNB - 1 PBMap\ThreadsNB - 1
PBMap\Redraw = #True PBMap\Redraw = #True
@@ -2750,8 +2769,8 @@ CompilerIf #PB_Compiler_IsMainFile
PBMap::InitPBMap(#Window_0) PBMap::InitPBMap(#Window_0)
PBMap::SetOption("ShowDegrees", "1") : Degrees = 0 PBMap::SetOption("ShowDegrees", "1") : Degrees = 0
PBMap::SetOption("ShowDebugInfos", "1") PBMap::SetOption("ShowDebugInfos", "1")
PBMap::SetDebugLevel(4) PBMap::SetDebugLevel(5)
PBMap::SetOption("Verbose", "0") PBMap::SetOption("Verbose", "1")
PBMap::SetOption("ShowScale", "1") PBMap::SetOption("ShowScale", "1")
PBMap::SetOption("Warning", "1") PBMap::SetOption("Warning", "1")
PBMap::SetOption("ShowMarkersLegend", "1") PBMap::SetOption("ShowMarkersLegend", "1")
@@ -2894,8 +2913,8 @@ CompilerIf #PB_Compiler_IsMainFile
CompilerEndIf CompilerEndIf
; IDE Options = PureBasic 5.60 (Windows - x64) ; IDE Options = PureBasic 5.60 (Windows - x64)
; CursorPosition = 2510 ; CursorPosition = 1181
; FirstLine = 2503 ; FirstLine = 1156
; Folding = ------------------- ; Folding = -------------------
; EnableThread ; EnableThread
; EnableXP ; EnableXP