File Search speed

  1. 9 months ago

    Hello,

    I have to search thru 2000 file names looking for a specific one and its really slow. I started with this code:

    If f <> Nil then
      u = f.Count
      for i = 1 to u
        if not f.item(i).Directory Then
          TheFileName = f.Item(i).DisplayName
          if Instr(TheFileName,BoardCards) > 0 Then FoundIt = 1
        end if
        if FoundIt = 1 then exit
      Next
    end

    but it took 32 seconds to complete the search.

    I then tried putting the filenames into an array first then searching but it still took 10 seconds:

    If f <> Nil then
      u = f.Count
      for i = 1 to u
        if not f.item(i).Directory Then
          TheFiles(i) = f.Item(i).DisplayName
        end if
      Next
    end
    
    for i  = 1 to u
      TheFileName = TheFiles(i)
      if Instr(TheFileName,BoardCards) > 0 Then FoundIt = 1
      if FoundIt = 1 then exit
    Next

    Any ideas on how to make this faster? This is really bogging my program down. Any help would be appreciated.

    Thanks

    from somewhere-I-dont-remember of this forum...

    Function ItemNames(extends f as folderItem, ReturnFileNames as boolean = true, ReturnFolderNames as boolean = true) As String()
      // Returns an array containing the names of child items
      // within the given folder item. Returns files and/or
      // folders, as directed by the input booleans.
      //
      // Returns an empty array if f doesn't exist, or if f
      // isn't a directory, or if both input booleans are false.
      //
      // The iteration is not recursive. On Windows, the special
      // directories "." and ".." are ignored.
      
      dim result() as string
      
      if f.Directory then
        
        #if TargetWin32
          
          // On Windows, RB's f.Item(i) is slow for folders with large
          // child item counts, so we use Declares on Windows instead.
          // Adapted from Aaron Ballman - see http://tinyurl.com/5susum
          
          Soft Declare Function FindFirstFileA Lib "Kernel32" (path as CString, data as Ptr) as Integer
          Soft Declare Function FindFirstFileW Lib "Kernel32" (path as WString, data as Ptr) as Integer
          Soft Declare Function FindNextFileA Lib "Kernel32" (handle as Integer, data as Ptr) as Boolean
          Soft Declare Function FindNextFileW Lib "Kernel32" (handle as Integer, data as Ptr) as Boolean
          Declare Sub FindClose Lib "Kernel32" (handle as Integer)
          
          dim UnicodeIsAvailable as boolean = System.IsFunctionAvailable("FindFirstFileW", "Kernel32")
          
          dim ChildData as MemoryBlock // WIN32_FIND_DATA struct
          dim ChildHandle as integer
          
          if UnicodeIsAvailable then
            
            ChildData = new MemoryBlock(592)
            ChildHandle = FindFirstFileW(f.AbsolutePath + "*.*", ChildData)
            
          else
            
            ChildData = new MemoryBlock(318)
            ChildHandle = FindFirstFileA(f.AbsolutePath + "*.*", ChildData)
            
          end if
          
          if ChildHandle <> -1 then
            
            dim ChildAttrs as UInt32 // first 4 bytes of WIN32_FIND_DATA
            dim ChildName as string
            
            // Loop through remaining items in the folder.
            
            dim FoundNextChild as Boolean
            
            do // loop through remaining children
              
              ChildAttrs = ChildData.UInt32Value(0)
              const NameOffset = 44
              
              if UnicodeIsAvailable then
                
                ChildName = ChildData.WString(NameOffset)
                FoundNextChild = FindNextFileW(ChildHandle, ChildData)
                
              else
                
                ChildName = ChildData.CString(NameOffset)
                FoundNextChild = FindNextFileA(ChildHandle, ChildData)
                
              end if
              
              // Now that we have its name and attributes, we can decide
              // whether this child should be added to our return array.
              
              dim ChildIsFolder as boolean = (ChildAttrs and UInt32(16)) <> 0
              
              if (ReturnFileNames and not ChildIsFolder) or _
                (ReturnFolderNames and ChildIsFolder) then
                
                if childName <> "." and childName <> ".." then
                  result.Append(ChildName)
                end if
                
              end if
              
            loop until not FoundNextChild // should really test GetLastError for ERROR_NO_MORE_FILES
            
            FindClose ChildHandle
            
          end if
          
        #else
          
          // On non-Windows systems, pure RB code seems pretty fast.
          
          dim child as FolderItem, ub as Integer = F.Count
          
          for i as integer = 1 to ub
            
            child = f.TrueItem(i)
            
            if (ReturnFileNames and not Child.Directory) or _
              (ReturnFolderNames and Child.Directory) then
              result.Append child.Name
            end if
            
          next i
          
        #endif
        
      end if
      
      return result
      
    End Function
  2. Jeff T

    10 Jan 2019 Pre-Release Testers Midlands of England, Europe

    Using the collection of files in Xojo's folderitem is slow.

    If you are on Windows, one technique I have seen used is to shell out to a command, and do a
    DIR > LISTOFFILES.TXT

    Then open LISTOFFILES.TXT as a textinput stream, reading the lot into a string.
    And use INSTR() on the string.

    or perhaps

    DIR *searchstring*.* > LISTOFFILES.TXT to get DOS to do the wildcard search for you!

    I imagine a similar technique is possible in OSX using a command such as ls

  3. Hi Jeff. I will try the shell wildcard search and report back. thanks for the info!

  4. Jean-Yves P

    10 Jan 2019 Pre-Release Testers, Xojo Pro Answer Europe (France, Besançon)

    from somewhere-I-dont-remember of this forum...

    Function ItemNames(extends f as folderItem, ReturnFileNames as boolean = true, ReturnFolderNames as boolean = true) As String()
      // Returns an array containing the names of child items
      // within the given folder item. Returns files and/or
      // folders, as directed by the input booleans.
      //
      // Returns an empty array if f doesn't exist, or if f
      // isn't a directory, or if both input booleans are false.
      //
      // The iteration is not recursive. On Windows, the special
      // directories "." and ".." are ignored.
      
      dim result() as string
      
      if f.Directory then
        
        #if TargetWin32
          
          // On Windows, RB's f.Item(i) is slow for folders with large
          // child item counts, so we use Declares on Windows instead.
          // Adapted from Aaron Ballman - see http://tinyurl.com/5susum
          
          Soft Declare Function FindFirstFileA Lib "Kernel32" (path as CString, data as Ptr) as Integer
          Soft Declare Function FindFirstFileW Lib "Kernel32" (path as WString, data as Ptr) as Integer
          Soft Declare Function FindNextFileA Lib "Kernel32" (handle as Integer, data as Ptr) as Boolean
          Soft Declare Function FindNextFileW Lib "Kernel32" (handle as Integer, data as Ptr) as Boolean
          Declare Sub FindClose Lib "Kernel32" (handle as Integer)
          
          dim UnicodeIsAvailable as boolean = System.IsFunctionAvailable("FindFirstFileW", "Kernel32")
          
          dim ChildData as MemoryBlock // WIN32_FIND_DATA struct
          dim ChildHandle as integer
          
          if UnicodeIsAvailable then
            
            ChildData = new MemoryBlock(592)
            ChildHandle = FindFirstFileW(f.AbsolutePath + "*.*", ChildData)
            
          else
            
            ChildData = new MemoryBlock(318)
            ChildHandle = FindFirstFileA(f.AbsolutePath + "*.*", ChildData)
            
          end if
          
          if ChildHandle <> -1 then
            
            dim ChildAttrs as UInt32 // first 4 bytes of WIN32_FIND_DATA
            dim ChildName as string
            
            // Loop through remaining items in the folder.
            
            dim FoundNextChild as Boolean
            
            do // loop through remaining children
              
              ChildAttrs = ChildData.UInt32Value(0)
              const NameOffset = 44
              
              if UnicodeIsAvailable then
                
                ChildName = ChildData.WString(NameOffset)
                FoundNextChild = FindNextFileW(ChildHandle, ChildData)
                
              else
                
                ChildName = ChildData.CString(NameOffset)
                FoundNextChild = FindNextFileA(ChildHandle, ChildData)
                
              end if
              
              // Now that we have its name and attributes, we can decide
              // whether this child should be added to our return array.
              
              dim ChildIsFolder as boolean = (ChildAttrs and UInt32(16)) <> 0
              
              if (ReturnFileNames and not ChildIsFolder) or _
                (ReturnFolderNames and ChildIsFolder) then
                
                if childName <> "." and childName <> ".." then
                  result.Append(ChildName)
                end if
                
              end if
              
            loop until not FoundNextChild // should really test GetLastError for ERROR_NO_MORE_FILES
            
            FindClose ChildHandle
            
          end if
          
        #else
          
          // On non-Windows systems, pure RB code seems pretty fast.
          
          dim child as FolderItem, ub as Integer = F.Count
          
          for i as integer = 1 to ub
            
            child = f.TrueItem(i)
            
            if (ReturnFileNames and not Child.Directory) or _
              (ReturnFolderNames and Child.Directory) then
              result.Append child.Name
            end if
            
          next i
          
        #endif
        
      end if
      
      return result
      
    End Function
  5. Hey! That method above was nearly instant! thanks for the code :-)

  6. Jean-Yves P

    10 Jan 2019 Pre-Release Testers, Xojo Pro Europe (France, Besançon)

    just seeing the code again, may be it will not work for 64 bits (windows) ...

  7. Andre K

    10 Jan 2019 Pre-Release Testers

    @Jean-YvesPochez just seeing the code again, may be it will not work for 64 bits (windows) ...

    Works just fine on 64 bits Windows 10, no problem at all.

  8. Jean-Yves P

    10 Jan 2019 Pre-Release Testers, Xojo Pro Europe (France, Besançon)

    the code came from here : https://forum.xojo.com/conversation/post/111418

or Sign Up to reply!